2024-11-27 11:34:54,375 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 11:34:54,386 main DEBUG Took 0.009386 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-27 11:34:54,386 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-27 11:34:54,386 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-27 11:34:54,387 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-27 11:34:54,388 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,395 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-27 11:34:54,406 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,408 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,408 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,409 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,409 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,410 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,410 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,411 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,411 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,411 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,412 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,412 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,413 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,414 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,414 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,414 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,415 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,415 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,416 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,416 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-27 11:34:54,416 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,416 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-27 11:34:54,418 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-27 11:34:54,419 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-27 11:34:54,421 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-27 11:34:54,421 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-27 11:34:54,422 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-27 11:34:54,422 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-27 11:34:54,431 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-27 11:34:54,433 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-27 11:34:54,434 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-27 11:34:54,435 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-27 11:34:54,435 main DEBUG createAppenders(={Console}) 2024-11-27 11:34:54,436 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-27 11:34:54,436 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-27 11:34:54,436 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-27 11:34:54,437 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-27 11:34:54,437 main DEBUG OutputStream closed 2024-11-27 11:34:54,437 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-27 11:34:54,437 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-27 11:34:54,438 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-27 11:34:54,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-27 11:34:54,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-27 11:34:54,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-27 11:34:54,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-27 11:34:54,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-27 11:34:54,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-27 11:34:54,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-27 11:34:54,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-27 11:34:54,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-27 11:34:54,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-27 11:34:54,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-27 11:34:54,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-27 11:34:54,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-27 11:34:54,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-27 11:34:54,509 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-27 11:34:54,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-27 11:34:54,510 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-27 11:34:54,511 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-27 11:34:54,513 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27 11:34:54,514 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-27 11:34:54,514 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-27 11:34:54,514 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-27T11:34:54,734 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059 2024-11-27 11:34:54,737 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-27 11:34:54,737 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-27T11:34:54,746 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-27T11:34:54,764 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-27T11:34:54,778 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d, deleteOnExit=true 2024-11-27T11:34:54,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-27T11:34:54,779 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/test.cache.data in system properties and HBase conf 2024-11-27T11:34:54,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/hadoop.tmp.dir in system properties and HBase conf 2024-11-27T11:34:54,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/hadoop.log.dir in system properties and HBase conf 2024-11-27T11:34:54,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-27T11:34:54,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-27T11:34:54,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-27T11:34:54,869 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-27T11:34:54,963 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-27T11:34:54,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-27T11:34:54,968 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-27T11:34:54,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-27T11:34:54,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T11:34:54,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-27T11:34:54,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-27T11:34:54,971 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-27T11:34:54,972 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T11:34:54,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-27T11:34:54,973 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/nfs.dump.dir in system properties and HBase conf 2024-11-27T11:34:54,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/java.io.tmpdir in system properties and HBase conf 2024-11-27T11:34:54,974 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-27T11:34:54,975 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-27T11:34:54,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-27T11:34:55,815 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-27T11:34:55,889 INFO [Time-limited test {}] log.Log(170): Logging initialized @2188ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-27T11:34:55,965 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T11:34:56,024 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T11:34:56,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T11:34:56,044 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T11:34:56,045 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-27T11:34:56,057 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T11:34:56,059 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@493a7020{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/hadoop.log.dir/,AVAILABLE} 2024-11-27T11:34:56,060 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4a4bbe01{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T11:34:56,265 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3654837{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/java.io.tmpdir/jetty-localhost-38873-hadoop-hdfs-3_4_1-tests_jar-_-any-13515861514563161126/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T11:34:56,271 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3edc1d8c{HTTP/1.1, (http/1.1)}{localhost:38873} 2024-11-27T11:34:56,272 INFO [Time-limited test {}] server.Server(415): Started @2571ms 2024-11-27T11:34:56,660 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-27T11:34:56,667 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-27T11:34:56,668 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-27T11:34:56,668 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-27T11:34:56,669 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-27T11:34:56,669 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5afe3a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/hadoop.log.dir/,AVAILABLE} 2024-11-27T11:34:56,670 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33cacd2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-27T11:34:56,790 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@486706b6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/java.io.tmpdir/jetty-localhost-39513-hadoop-hdfs-3_4_1-tests_jar-_-any-634312364509723353/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T11:34:56,791 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1889dd8{HTTP/1.1, (http/1.1)}{localhost:39513} 2024-11-27T11:34:56,791 INFO [Time-limited test {}] server.Server(415): Started @3090ms 2024-11-27T11:34:56,846 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-27T11:34:57,338 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/data/data2/current/BP-972261524-172.17.0.2-1732707295566/current, will proceed with Du for space computation calculation, 2024-11-27T11:34:57,338 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/data/data1/current/BP-972261524-172.17.0.2-1732707295566/current, will proceed with Du for space computation calculation, 2024-11-27T11:34:57,377 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-27T11:34:57,433 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x26e1dca5e474c98e with lease ID 0xcc57eee9cb625721: Processing first storage report for DS-35c63855-56da-4065-8497-23def7eefd75 from datanode DatanodeRegistration(127.0.0.1:37081, datanodeUuid=b9f6a65c-eef6-4672-a6a3-56fdb08a19fe, infoPort=42953, infoSecurePort=0, ipcPort=42929, storageInfo=lv=-57;cid=testClusterID;nsid=291446865;c=1732707295566) 2024-11-27T11:34:57,435 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26e1dca5e474c98e with lease ID 0xcc57eee9cb625721: from storage DS-35c63855-56da-4065-8497-23def7eefd75 node DatanodeRegistration(127.0.0.1:37081, datanodeUuid=b9f6a65c-eef6-4672-a6a3-56fdb08a19fe, infoPort=42953, infoSecurePort=0, ipcPort=42929, storageInfo=lv=-57;cid=testClusterID;nsid=291446865;c=1732707295566), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-27T11:34:57,435 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x26e1dca5e474c98e with lease ID 0xcc57eee9cb625721: Processing first storage report for DS-148f9bac-6d77-4f9f-b61f-1627da45395c from datanode DatanodeRegistration(127.0.0.1:37081, datanodeUuid=b9f6a65c-eef6-4672-a6a3-56fdb08a19fe, infoPort=42953, infoSecurePort=0, ipcPort=42929, storageInfo=lv=-57;cid=testClusterID;nsid=291446865;c=1732707295566) 2024-11-27T11:34:57,436 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x26e1dca5e474c98e with lease ID 0xcc57eee9cb625721: from storage DS-148f9bac-6d77-4f9f-b61f-1627da45395c node DatanodeRegistration(127.0.0.1:37081, datanodeUuid=b9f6a65c-eef6-4672-a6a3-56fdb08a19fe, infoPort=42953, infoSecurePort=0, ipcPort=42929, storageInfo=lv=-57;cid=testClusterID;nsid=291446865;c=1732707295566), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-27T11:34:57,436 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059 2024-11-27T11:34:57,516 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/zookeeper_0, clientPort=50523, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-27T11:34:57,525 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50523 2024-11-27T11:34:57,535 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:57,538 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:57,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741825_1001 (size=7) 2024-11-27T11:34:58,157 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b with version=8 2024-11-27T11:34:58,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/hbase-staging 2024-11-27T11:34:58,251 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-27T11:34:58,496 INFO [Time-limited test {}] client.ConnectionUtils(128): master/5b617c7edc8e:0 server-side Connection retries=45 2024-11-27T11:34:58,506 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T11:34:58,507 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T11:34:58,511 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T11:34:58,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T11:34:58,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T11:34:58,648 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-27T11:34:58,708 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-27T11:34:58,717 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-27T11:34:58,721 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T11:34:58,750 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20069 (auto-detected) 2024-11-27T11:34:58,751 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-27T11:34:58,770 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35539 2024-11-27T11:34:58,801 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35539 connecting to ZooKeeper ensemble=127.0.0.1:50523 2024-11-27T11:34:58,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:355390x0, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T11:34:58,843 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35539-0x101593513670000 connected 2024-11-27T11:34:58,878 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:58,881 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:58,890 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T11:34:58,896 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b, hbase.cluster.distributed=false 2024-11-27T11:34:58,927 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T11:34:58,933 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35539 2024-11-27T11:34:58,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35539 2024-11-27T11:34:58,934 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35539 2024-11-27T11:34:58,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35539 2024-11-27T11:34:58,935 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35539 2024-11-27T11:34:59,045 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/5b617c7edc8e:0 server-side Connection retries=45 2024-11-27T11:34:59,047 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T11:34:59,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-27T11:34:59,048 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-27T11:34:59,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-27T11:34:59,048 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-27T11:34:59,051 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-27T11:34:59,053 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-27T11:34:59,054 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35461 2024-11-27T11:34:59,056 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35461 connecting to ZooKeeper ensemble=127.0.0.1:50523 2024-11-27T11:34:59,057 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:59,060 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:59,067 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354610x0, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-27T11:34:59,068 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35461-0x101593513670001 connected 2024-11-27T11:34:59,069 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T11:34:59,073 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-27T11:34:59,081 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-27T11:34:59,083 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-27T11:34:59,088 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-27T11:34:59,088 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35461 2024-11-27T11:34:59,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35461 2024-11-27T11:34:59,089 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35461 2024-11-27T11:34:59,090 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35461 2024-11-27T11:34:59,090 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35461 2024-11-27T11:34:59,105 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;5b617c7edc8e:35539 2024-11-27T11:34:59,106 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/5b617c7edc8e,35539,1732707298302 2024-11-27T11:34:59,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T11:34:59,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T11:34:59,114 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/5b617c7edc8e,35539,1732707298302 2024-11-27T11:34:59,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-27T11:34:59,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:34:59,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:34:59,139 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-27T11:34:59,140 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/5b617c7edc8e,35539,1732707298302 from backup master directory 2024-11-27T11:34:59,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T11:34:59,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/5b617c7edc8e,35539,1732707298302 2024-11-27T11:34:59,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-27T11:34:59,143 WARN [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T11:34:59,144 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=5b617c7edc8e,35539,1732707298302 2024-11-27T11:34:59,146 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-27T11:34:59,147 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-27T11:34:59,203 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/hbase.id] with ID: 298a1ba5-84a0-4a11-841c-c5777d1b0342 2024-11-27T11:34:59,203 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/.tmp/hbase.id 2024-11-27T11:34:59,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741826_1002 (size=42) 2024-11-27T11:34:59,615 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/.tmp/hbase.id]:[hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/hbase.id] 2024-11-27T11:34:59,655 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-27T11:34:59,659 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-27T11:34:59,678 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-27T11:34:59,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:34:59,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:34:59,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741827_1003 (size=196) 2024-11-27T11:35:00,113 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:35:00,117 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-27T11:35:00,139 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:00,145 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T11:35:00,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741828_1004 (size=1189) 2024-11-27T11:35:00,595 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store 2024-11-27T11:35:00,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741829_1005 (size=34) 2024-11-27T11:35:01,019 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-27T11:35:01,022 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:01,023 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T11:35:01,023 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:35:01,023 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:35:01,025 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T11:35:01,025 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:35:01,025 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:35:01,026 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732707301023Disabling compacts and flushes for region at 1732707301023Disabling writes for close at 1732707301025 (+2 ms)Writing region close event to WAL at 1732707301025Closed at 1732707301025 2024-11-27T11:35:01,028 WARN [master/5b617c7edc8e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/.initializing 2024-11-27T11:35:01,028 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/WALs/5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:01,037 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T11:35:01,052 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5b617c7edc8e%2C35539%2C1732707298302, suffix=, logDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/WALs/5b617c7edc8e,35539,1732707298302, archiveDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/oldWALs, maxLogs=10 2024-11-27T11:35:01,083 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/WALs/5b617c7edc8e,35539,1732707298302/5b617c7edc8e%2C35539%2C1732707298302.1732707301059, exclude list is [], retry=0 2024-11-27T11:35:01,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37081,DS-35c63855-56da-4065-8497-23def7eefd75,DISK] 2024-11-27T11:35:01,102 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-27T11:35:01,138 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/WALs/5b617c7edc8e,35539,1732707298302/5b617c7edc8e%2C35539%2C1732707298302.1732707301059 2024-11-27T11:35:01,139 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42953:42953)] 2024-11-27T11:35:01,140 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:35:01,140 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:01,143 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,144 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,207 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-27T11:35:01,211 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:01,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:01,214 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-27T11:35:01,218 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:01,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:01,220 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-27T11:35:01,222 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:01,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:01,224 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-27T11:35:01,226 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:01,227 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:01,228 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,231 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,233 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,238 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,240 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,244 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-27T11:35:01,249 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-27T11:35:01,253 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:35:01,255 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65023325, jitterRate=-0.031076952815055847}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-27T11:35:01,261 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732707301156Initializing all the Stores at 1732707301158 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707301159 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707301159Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707301160 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707301160Cleaning up temporary data from old regions at 1732707301240 (+80 ms)Region opened successfully at 1732707301261 (+21 ms) 2024-11-27T11:35:01,262 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-27T11:35:01,296 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8413430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5b617c7edc8e/172.17.0.2:0 2024-11-27T11:35:01,327 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-27T11:35:01,339 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-27T11:35:01,339 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-27T11:35:01,342 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-27T11:35:01,343 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-27T11:35:01,348 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-27T11:35:01,349 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-27T11:35:01,374 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-27T11:35:01,382 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-27T11:35:01,384 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-27T11:35:01,386 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-27T11:35:01,388 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-27T11:35:01,395 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-27T11:35:01,397 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-27T11:35:01,400 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-27T11:35:01,402 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-27T11:35:01,403 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-27T11:35:01,404 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-27T11:35:01,423 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-27T11:35:01,424 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-27T11:35:01,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T11:35:01,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-27T11:35:01,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:01,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:01,433 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=5b617c7edc8e,35539,1732707298302, sessionid=0x101593513670000, setting cluster-up flag (Was=false) 2024-11-27T11:35:01,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:01,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:01,453 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-27T11:35:01,455 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:01,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:01,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:01,466 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-27T11:35:01,468 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:01,474 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-27T11:35:01,495 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(746): ClusterId : 298a1ba5-84a0-4a11-841c-c5777d1b0342 2024-11-27T11:35:01,498 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-27T11:35:01,504 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-27T11:35:01,504 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-27T11:35:01,509 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-27T11:35:01,509 DEBUG [RS:0;5b617c7edc8e:35461 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6832103e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=5b617c7edc8e/172.17.0.2:0 2024-11-27T11:35:01,529 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;5b617c7edc8e:35461 2024-11-27T11:35:01,533 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-27T11:35:01,533 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-27T11:35:01,533 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-27T11:35:01,536 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(2659): reportForDuty to master=5b617c7edc8e,35539,1732707298302 with port=35461, startcode=1732707299005 2024-11-27T11:35:01,550 DEBUG [RS:0;5b617c7edc8e:35461 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-27T11:35:01,550 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-27T11:35:01,560 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-27T11:35:01,567 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-27T11:35:01,572 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 5b617c7edc8e,35539,1732707298302 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-27T11:35:01,580 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/5b617c7edc8e:0, corePoolSize=5, maxPoolSize=5 2024-11-27T11:35:01,580 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/5b617c7edc8e:0, corePoolSize=5, maxPoolSize=5 2024-11-27T11:35:01,580 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/5b617c7edc8e:0, corePoolSize=5, maxPoolSize=5 2024-11-27T11:35:01,581 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/5b617c7edc8e:0, corePoolSize=5, maxPoolSize=5 2024-11-27T11:35:01,581 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/5b617c7edc8e:0, corePoolSize=10, maxPoolSize=10 2024-11-27T11:35:01,581 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,581 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/5b617c7edc8e:0, corePoolSize=2, maxPoolSize=2 2024-11-27T11:35:01,581 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,582 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732707331582 2024-11-27T11:35:01,584 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-27T11:35:01,585 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-27T11:35:01,587 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T11:35:01,588 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-27T11:35:01,588 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-27T11:35:01,590 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-27T11:35:01,590 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-27T11:35:01,590 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-27T11:35:01,591 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-27T11:35:01,592 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,595 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:01,595 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-27T11:35:01,595 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35539 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:01,597 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-27T11:35:01,598 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-27T11:35:01,598 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-27T11:35:01,600 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-27T11:35:01,601 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-27T11:35:01,603 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/5b617c7edc8e:0:becomeActiveMaster-HFileCleaner.large.0-1732707301602,5,FailOnTimeoutGroup] 2024-11-27T11:35:01,604 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/5b617c7edc8e:0:becomeActiveMaster-HFileCleaner.small.0-1732707301604,5,FailOnTimeoutGroup] 2024-11-27T11:35:01,604 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,605 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-27T11:35:01,606 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,606 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741831_1007 (size=1321) 2024-11-27T11:35:01,624 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-27T11:35:01,624 WARN [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-27T11:35:01,726 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(2659): reportForDuty to master=5b617c7edc8e,35539,1732707298302 with port=35461, startcode=1732707299005 2024-11-27T11:35:01,728 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35539 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:01,730 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35539 {}] master.ServerManager(517): Registering regionserver=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:01,739 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:35:01,739 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42541 2024-11-27T11:35:01,739 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-27T11:35:01,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T11:35:01,744 DEBUG [RS:0;5b617c7edc8e:35461 {}] zookeeper.ZKUtil(111): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:01,744 WARN [RS:0;5b617c7edc8e:35461 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-27T11:35:01,744 INFO [RS:0;5b617c7edc8e:35461 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T11:35:01,745 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:01,747 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [5b617c7edc8e,35461,1732707299005] 2024-11-27T11:35:01,771 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-27T11:35:01,785 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-27T11:35:01,790 INFO [RS:0;5b617c7edc8e:35461 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-27T11:35:01,790 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,791 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-27T11:35:01,796 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-27T11:35:01,798 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,798 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,798 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,798 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,798 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,799 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,799 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/5b617c7edc8e:0, corePoolSize=2, maxPoolSize=2 2024-11-27T11:35:01,799 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,799 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,799 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,799 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,800 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,800 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/5b617c7edc8e:0, corePoolSize=1, maxPoolSize=1 2024-11-27T11:35:01,800 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/5b617c7edc8e:0, corePoolSize=3, maxPoolSize=3 2024-11-27T11:35:01,800 DEBUG [RS:0;5b617c7edc8e:35461 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0, corePoolSize=3, maxPoolSize=3 2024-11-27T11:35:01,801 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,801 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,801 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,801 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,802 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,802 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35461,1732707299005-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T11:35:01,823 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-27T11:35:01,825 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35461,1732707299005-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,825 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,826 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.Replication(171): 5b617c7edc8e,35461,1732707299005 started 2024-11-27T11:35:01,845 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:01,845 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1482): Serving as 5b617c7edc8e,35461,1732707299005, RpcServer on 5b617c7edc8e/172.17.0.2:35461, sessionid=0x101593513670001 2024-11-27T11:35:01,846 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-27T11:35:01,846 DEBUG [RS:0;5b617c7edc8e:35461 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:01,846 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5b617c7edc8e,35461,1732707299005' 2024-11-27T11:35:01,847 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-27T11:35:01,848 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-27T11:35:01,848 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-27T11:35:01,848 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-27T11:35:01,849 DEBUG [RS:0;5b617c7edc8e:35461 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:01,849 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '5b617c7edc8e,35461,1732707299005' 2024-11-27T11:35:01,849 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-27T11:35:01,849 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-27T11:35:01,850 DEBUG [RS:0;5b617c7edc8e:35461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-27T11:35:01,850 INFO [RS:0;5b617c7edc8e:35461 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-27T11:35:01,850 INFO [RS:0;5b617c7edc8e:35461 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-27T11:35:01,956 INFO [RS:0;5b617c7edc8e:35461 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-27T11:35:01,959 INFO [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5b617c7edc8e%2C35461%2C1732707299005, suffix=, logDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005, archiveDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/oldWALs, maxLogs=32 2024-11-27T11:35:01,976 DEBUG [RS:0;5b617c7edc8e:35461 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005/5b617c7edc8e%2C35461%2C1732707299005.1732707301961, exclude list is [], retry=0 2024-11-27T11:35:01,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37081,DS-35c63855-56da-4065-8497-23def7eefd75,DISK] 2024-11-27T11:35:01,984 INFO [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005/5b617c7edc8e%2C35461%2C1732707299005.1732707301961 2024-11-27T11:35:01,984 DEBUG [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42953:42953)] 2024-11-27T11:35:02,009 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-27T11:35:02,009 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:35:02,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741833_1009 (size=32) 2024-11-27T11:35:02,421 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:02,424 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T11:35:02,426 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T11:35:02,426 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,427 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,428 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T11:35:02,430 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T11:35:02,430 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,431 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T11:35:02,433 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T11:35:02,433 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,434 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T11:35:02,437 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T11:35:02,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,438 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T11:35:02,439 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740 2024-11-27T11:35:02,440 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740 2024-11-27T11:35:02,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T11:35:02,443 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T11:35:02,443 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:35:02,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T11:35:02,449 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:35:02,450 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71158946, jitterRate=0.0603509247303009}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:35:02,452 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732707302421Initializing all the Stores at 1732707302423 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707302423Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707302423Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707302423Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707302423Cleaning up temporary data from old regions at 1732707302443 (+20 ms)Region opened successfully at 1732707302452 (+9 ms) 2024-11-27T11:35:02,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T11:35:02,453 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T11:35:02,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T11:35:02,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T11:35:02,453 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T11:35:02,454 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T11:35:02,454 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732707302452Disabling compacts and flushes for region at 1732707302453 (+1 ms)Disabling writes for close at 1732707302453Writing region close event to WAL at 1732707302454 (+1 ms)Closed at 1732707302454 2024-11-27T11:35:02,458 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T11:35:02,458 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-27T11:35:02,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-27T11:35:02,472 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T11:35:02,475 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-27T11:35:02,628 DEBUG [5b617c7edc8e:35539 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-27T11:35:02,640 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:02,647 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5b617c7edc8e,35461,1732707299005, state=OPENING 2024-11-27T11:35:02,653 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-27T11:35:02,654 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:02,655 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:35:02,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T11:35:02,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T11:35:02,657 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-27T11:35:02,659 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:02,836 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-27T11:35:02,839 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46391, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-27T11:35:02,850 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-27T11:35:02,851 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-27T11:35:02,851 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-27T11:35:02,855 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=5b617c7edc8e%2C35461%2C1732707299005.meta, suffix=.meta, logDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005, archiveDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/oldWALs, maxLogs=32 2024-11-27T11:35:02,872 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005/5b617c7edc8e%2C35461%2C1732707299005.meta.1732707302857.meta, exclude list is [], retry=0 2024-11-27T11:35:02,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37081,DS-35c63855-56da-4065-8497-23def7eefd75,DISK] 2024-11-27T11:35:02,878 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/WALs/5b617c7edc8e,35461,1732707299005/5b617c7edc8e%2C35461%2C1732707299005.meta.1732707302857.meta 2024-11-27T11:35:02,879 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42953:42953)] 2024-11-27T11:35:02,879 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:35:02,881 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-27T11:35:02,884 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-27T11:35:02,888 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-27T11:35:02,893 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-27T11:35:02,893 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:02,893 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-27T11:35:02,893 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-27T11:35:02,897 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-27T11:35:02,898 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-27T11:35:02,898 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,899 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-27T11:35:02,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-27T11:35:02,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,902 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-27T11:35:02,904 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-27T11:35:02,904 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-27T11:35:02,907 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-27T11:35:02,907 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:02,908 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-27T11:35:02,908 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-27T11:35:02,909 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740 2024-11-27T11:35:02,912 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740 2024-11-27T11:35:02,914 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-27T11:35:02,914 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-27T11:35:02,915 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:35:02,917 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-27T11:35:02,919 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70763371, jitterRate=0.05445639789104462}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:35:02,919 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-27T11:35:02,921 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732707302894Writing region info on filesystem at 1732707302894Initializing all the Stores at 1732707302896 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707302896Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707302896Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707302896Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732707302896Cleaning up temporary data from old regions at 1732707302914 (+18 ms)Running coprocessor post-open hooks at 1732707302919 (+5 ms)Region opened successfully at 1732707302920 (+1 ms) 2024-11-27T11:35:02,928 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732707302826 2024-11-27T11:35:02,940 DEBUG [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-27T11:35:02,941 INFO [RS_OPEN_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-27T11:35:02,943 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:02,945 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 5b617c7edc8e,35461,1732707299005, state=OPEN 2024-11-27T11:35:02,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T11:35:02,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-27T11:35:02,956 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T11:35:02,956 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-27T11:35:02,957 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:02,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-27T11:35:02,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=5b617c7edc8e,35461,1732707299005 in 298 msec 2024-11-27T11:35:02,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-27T11:35:02,969 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 500 msec 2024-11-27T11:35:02,970 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-27T11:35:02,970 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-27T11:35:02,991 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:02,992 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:03,014 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:03,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48285, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:03,038 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.5340 sec 2024-11-27T11:35:03,039 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732707303038, completionTime=-1 2024-11-27T11:35:03,042 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-27T11:35:03,042 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-27T11:35:03,070 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-27T11:35:03,070 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732707363070 2024-11-27T11:35:03,070 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732707423070 2024-11-27T11:35:03,070 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-27T11:35:03,072 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35539,1732707298302-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,073 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35539,1732707298302-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,073 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35539,1732707298302-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,074 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-5b617c7edc8e:35539, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,075 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,075 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,082 DEBUG [master/5b617c7edc8e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-27T11:35:03,103 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.959sec 2024-11-27T11:35:03,104 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-27T11:35:03,106 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-27T11:35:03,107 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-27T11:35:03,107 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-27T11:35:03,107 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-27T11:35:03,108 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35539,1732707298302-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-27T11:35:03,109 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35539,1732707298302-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-27T11:35:03,117 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-27T11:35:03,118 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-27T11:35:03,118 INFO [master/5b617c7edc8e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=5b617c7edc8e,35539,1732707298302-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-27T11:35:03,207 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1b2839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:03,209 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-27T11:35:03,209 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-27T11:35:03,213 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:03,215 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:03,224 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:03,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:03,227 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b5c7a62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:03,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:03,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:03,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:03,235 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:03,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23ba7bf9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:03,239 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:03,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:03,250 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:03,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:03,284 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=237, ProcessCount=11, AvailableMemoryMB=7102 2024-11-27T11:35:03,309 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:35:03,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:03,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35874ff9 2024-11-27T11:35:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:35:03,321 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34244, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:35:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:35:03,333 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:35:03,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:03,345 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T11:35:03,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-27T11:35:03,347 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:03,349 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T11:35:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T11:35:03,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741835_1011 (size=959) 2024-11-27T11:35:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T11:35:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T11:35:03,766 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:35:03,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741836_1012 (size=53) 2024-11-27T11:35:03,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T11:35:04,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:04,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 080ce2238710db49165afb3a1a396a6b, disabling compactions & flushes 2024-11-27T11:35:04,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. after waiting 0 ms 2024-11-27T11:35:04,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 080ce2238710db49165afb3a1a396a6b: Waiting for close lock at 1732707304176Disabling compacts and flushes for region at 1732707304176Disabling writes for close at 1732707304176Writing region close event to WAL at 1732707304176Closed at 1732707304176 2024-11-27T11:35:04,178 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T11:35:04,184 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732707304179"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732707304179"}]},"ts":"1732707304179"} 2024-11-27T11:35:04,188 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T11:35:04,190 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T11:35:04,193 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707304190"}]},"ts":"1732707304190"} 2024-11-27T11:35:04,198 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T11:35:04,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, ASSIGN}] 2024-11-27T11:35:04,202 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, ASSIGN 2024-11-27T11:35:04,204 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, ASSIGN; state=OFFLINE, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=false 2024-11-27T11:35:04,355 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=080ce2238710db49165afb3a1a396a6b, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:04,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, ASSIGN because future has completed 2024-11-27T11:35:04,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:04,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T11:35:04,522 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,522 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:35:04,523 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,523 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:04,523 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,523 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,526 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,528 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080ce2238710db49165afb3a1a396a6b columnFamilyName A 2024-11-27T11:35:04,528 DEBUG [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:04,529 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] regionserver.HStore(327): Store=080ce2238710db49165afb3a1a396a6b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:04,529 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,531 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080ce2238710db49165afb3a1a396a6b columnFamilyName B 2024-11-27T11:35:04,531 DEBUG [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:04,532 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] regionserver.HStore(327): Store=080ce2238710db49165afb3a1a396a6b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:04,532 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,534 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 080ce2238710db49165afb3a1a396a6b columnFamilyName C 2024-11-27T11:35:04,534 DEBUG [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:04,535 INFO [StoreOpener-080ce2238710db49165afb3a1a396a6b-1 {}] regionserver.HStore(327): Store=080ce2238710db49165afb3a1a396a6b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:04,535 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,538 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,540 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,540 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,541 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:35:04,545 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,549 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:35:04,550 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 080ce2238710db49165afb3a1a396a6b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69625366, jitterRate=0.037498801946640015}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:35:04,550 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:04,551 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 080ce2238710db49165afb3a1a396a6b: Running coprocessor pre-open hook at 1732707304524Writing region info on filesystem at 1732707304524Initializing all the Stores at 1732707304525 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707304525Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707304525Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707304526 (+1 ms)Cleaning up temporary data from old regions at 1732707304540 (+14 ms)Running coprocessor post-open hooks at 1732707304550 (+10 ms)Region opened successfully at 1732707304551 (+1 ms) 2024-11-27T11:35:04,554 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., pid=6, masterSystemTime=1732707304515 2024-11-27T11:35:04,558 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,558 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:04,559 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=080ce2238710db49165afb3a1a396a6b, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:04,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:35:04,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-27T11:35:04,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 in 204 msec 2024-11-27T11:35:04,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-27T11:35:04,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, ASSIGN in 370 msec 2024-11-27T11:35:04,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T11:35:04,576 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707304576"}]},"ts":"1732707304576"} 2024-11-27T11:35:04,579 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T11:35:04,581 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T11:35:04,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2440 sec 2024-11-27T11:35:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-27T11:35:05,504 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:05,510 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7617bcf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,510 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,512 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c6058d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,517 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e5dcf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,517 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,519 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,521 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,523 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:35:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:05,525 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10ab891d 2024-11-27T11:35:05,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:35:05,530 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:35:05,533 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@539c07a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,534 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,534 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,535 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14496fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,536 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,538 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34286, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@484f4e79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,541 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4c9506, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,544 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,545 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36644156, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,546 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4650d4b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,546 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,546 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,547 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9c1aaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,550 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3937ace3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,552 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22c400b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,552 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,554 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ffd0298, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,555 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615aad25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3289f0ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,559 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5805ae29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d230d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a334e4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af5f6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34362, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d37adaa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69b84aa0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,568 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,569 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,571 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3b3fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,572 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@482f98dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,572 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,572 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,573 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6e851d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,575 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34396, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29434e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,577 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1688b137, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,577 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:05,577 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:05,577 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68732d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:05,579 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34404, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:05,580 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce716ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:05,584 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,584 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,584 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,584 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,586 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,587 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,589 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,591 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,592 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,593 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,596 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,598 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,608 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:05,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:05,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-27T11:35:05,612 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:05,614 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:05,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:35:05,616 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:05,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:05,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:05,677 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:05,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:05,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:05,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:05,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:05,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:05,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/0574e36323da479ba5934d72ad2e1409 is 50, key is test_row_0/A:col10/1732707305666/Put/seqid=0 2024-11-27T11:35:05,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:05,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:05,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741837_1013 (size=12001) 2024-11-27T11:35:05,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/0574e36323da479ba5934d72ad2e1409 2024-11-27T11:35:05,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707365825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707365828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707365835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707365840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707365841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/154896dc1fc844448e92de5effeac903 is 50, key is test_row_0/B:col10/1732707305666/Put/seqid=0 2024-11-27T11:35:05,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741838_1014 (size=12001) 2024-11-27T11:35:05,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:05,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707365954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707365955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707365965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707365965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:05,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:05,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707365969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:05,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,971 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:05,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:05,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:06,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707366163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707366173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707366174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,176 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,177 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707366183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707366184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:06,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:06,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/154896dc1fc844448e92de5effeac903 2024-11-27T11:35:06,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/6705caa12e3f41c387922e628d8293c8 is 50, key is test_row_0/C:col10/1732707305666/Put/seqid=0 2024-11-27T11:35:06,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741839_1015 (size=12001) 2024-11-27T11:35:06,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/6705caa12e3f41c387922e628d8293c8 2024-11-27T11:35:06,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/0574e36323da479ba5934d72ad2e1409 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0574e36323da479ba5934d72ad2e1409 2024-11-27T11:35:06,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0574e36323da479ba5934d72ad2e1409, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T11:35:06,442 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-27T11:35:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/154896dc1fc844448e92de5effeac903 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/154896dc1fc844448e92de5effeac903 2024-11-27T11:35:06,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/154896dc1fc844448e92de5effeac903, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T11:35:06,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:06,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/6705caa12e3f41c387922e628d8293c8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6705caa12e3f41c387922e628d8293c8 2024-11-27T11:35:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6705caa12e3f41c387922e628d8293c8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-27T11:35:06,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 080ce2238710db49165afb3a1a396a6b in 805ms, sequenceid=13, compaction requested=false 2024-11-27T11:35:06,491 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T11:35:06,493 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:06,495 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:06,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T11:35:06,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/cc35374792ea4a41b7821a3ed36efe21 is 50, key is test_row_0/A:col10/1732707306491/Put/seqid=0 2024-11-27T11:35:06,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741840_1016 (size=16681) 2024-11-27T11:35:06,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707366516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707366520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707366516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707366524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707366529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:06,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,617 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707366643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707366645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707366645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707366646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,656 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707366650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:06,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:06,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:06,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707366864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707366866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707366866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707366865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:06,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707366873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,880 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:06,881 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:06,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/cc35374792ea4a41b7821a3ed36efe21 2024-11-27T11:35:06,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:06,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:06,939 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:06,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a6b3a011509148aaa650a79252de79ea is 50, key is test_row_0/B:col10/1732707306491/Put/seqid=0 2024-11-27T11:35:06,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741841_1017 (size=12001) 2024-11-27T11:35:06,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a6b3a011509148aaa650a79252de79ea 2024-11-27T11:35:07,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2465cba93a3f479e93073e61934525d4 is 50, key is test_row_0/C:col10/1732707306491/Put/seqid=0 2024-11-27T11:35:07,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741842_1018 (size=12001) 2024-11-27T11:35:07,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2465cba93a3f479e93073e61934525d4 2024-11-27T11:35:07,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/cc35374792ea4a41b7821a3ed36efe21 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/cc35374792ea4a41b7821a3ed36efe21 2024-11-27T11:35:07,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/cc35374792ea4a41b7821a3ed36efe21, entries=250, sequenceid=40, filesize=16.3 K 2024-11-27T11:35:07,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a6b3a011509148aaa650a79252de79ea as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a6b3a011509148aaa650a79252de79ea 2024-11-27T11:35:07,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a6b3a011509148aaa650a79252de79ea, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:35:07,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2465cba93a3f479e93073e61934525d4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2465cba93a3f479e93073e61934525d4 2024-11-27T11:35:07,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:07,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2465cba93a3f479e93073e61934525d4, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:35:07,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 080ce2238710db49165afb3a1a396a6b in 620ms, sequenceid=40, compaction requested=false 2024-11-27T11:35:07,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:07,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:07,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:07,182 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:07,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/d88bfcfc59f0469783c2696e6f8987a7 is 50, key is test_row_0/A:col10/1732707307179/Put/seqid=0 2024-11-27T11:35:07,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741843_1019 (size=14341) 2024-11-27T11:35:07,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/d88bfcfc59f0469783c2696e6f8987a7 2024-11-27T11:35:07,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:07,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5795e02db8ec44b9bc5771288cd6377e is 50, key is test_row_0/B:col10/1732707307179/Put/seqid=0 2024-11-27T11:35:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707367252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707367254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707367259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707367248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,268 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707367263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,269 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,273 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741844_1020 (size=12001) 2024-11-27T11:35:07,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5795e02db8ec44b9bc5771288cd6377e 2024-11-27T11:35:07,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/eaff47fd114e4f77bfe3c7ee676632e3 is 50, key is test_row_0/C:col10/1732707307179/Put/seqid=0 2024-11-27T11:35:07,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741845_1021 (size=12001) 2024-11-27T11:35:07,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/eaff47fd114e4f77bfe3c7ee676632e3 2024-11-27T11:35:07,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/d88bfcfc59f0469783c2696e6f8987a7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d88bfcfc59f0469783c2696e6f8987a7 2024-11-27T11:35:07,348 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d88bfcfc59f0469783c2696e6f8987a7, entries=200, sequenceid=52, filesize=14.0 K 2024-11-27T11:35:07,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5795e02db8ec44b9bc5771288cd6377e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5795e02db8ec44b9bc5771288cd6377e 2024-11-27T11:35:07,363 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5795e02db8ec44b9bc5771288cd6377e, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T11:35:07,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/eaff47fd114e4f77bfe3c7ee676632e3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/eaff47fd114e4f77bfe3c7ee676632e3 2024-11-27T11:35:07,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707367374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707367374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,378 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707367374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,379 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707367375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/eaff47fd114e4f77bfe3c7ee676632e3, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T11:35:07,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 080ce2238710db49165afb3a1a396a6b in 205ms, sequenceid=52, compaction requested=true 2024-11-27T11:35:07,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:07,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:07,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:07,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:07,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:07,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:07,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T11:35:07,395 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:07,395 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:35:07,398 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:07,398 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:07,402 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:07,404 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:07,404 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:07,405 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:07,406 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,407 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/154896dc1fc844448e92de5effeac903, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a6b3a011509148aaa650a79252de79ea, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5795e02db8ec44b9bc5771288cd6377e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.2 K 2024-11-27T11:35:07,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:07,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,411 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,413 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 154896dc1fc844448e92de5effeac903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732707305666 2024-11-27T11:35:07,414 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0574e36323da479ba5934d72ad2e1409, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/cc35374792ea4a41b7821a3ed36efe21, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d88bfcfc59f0469783c2696e6f8987a7] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=42.0 K 2024-11-27T11:35:07,415 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a6b3a011509148aaa650a79252de79ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707305817 2024-11-27T11:35:07,416 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5795e02db8ec44b9bc5771288cd6377e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707306515 2024-11-27T11:35:07,419 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0574e36323da479ba5934d72ad2e1409, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732707305666 2024-11-27T11:35:07,419 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc35374792ea4a41b7821a3ed36efe21, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707305817 2024-11-27T11:35:07,420 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d88bfcfc59f0469783c2696e6f8987a7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707306515 2024-11-27T11:35:07,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/a7ef85be98204c5dad438c140c896a68 is 50, key is test_row_0/A:col10/1732707307388/Put/seqid=0 2024-11-27T11:35:07,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741846_1022 (size=12001) 2024-11-27T11:35:07,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/a7ef85be98204c5dad438c140c896a68 2024-11-27T11:35:07,477 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#10 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:07,478 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/6d8b7c5bba7742f0817809c5d3b5c267 is 50, key is test_row_0/B:col10/1732707307179/Put/seqid=0 2024-11-27T11:35:07,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707367479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,491 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#11 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,492 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/57e760a789bb4a4791a85e07275bba1f is 50, key is test_row_0/A:col10/1732707307179/Put/seqid=0 2024-11-27T11:35:07,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741847_1023 (size=12104) 2024-11-27T11:35:07,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/6839c35c433849349dfd63dafad821b1 is 50, key is test_row_0/B:col10/1732707307388/Put/seqid=0 2024-11-27T11:35:07,523 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/6d8b7c5bba7742f0817809c5d3b5c267 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6d8b7c5bba7742f0817809c5d3b5c267 2024-11-27T11:35:07,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741848_1024 (size=12104) 2024-11-27T11:35:07,541 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/57e760a789bb4a4791a85e07275bba1f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/57e760a789bb4a4791a85e07275bba1f 2024-11-27T11:35:07,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741849_1025 (size=12001) 2024-11-27T11:35:07,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/6839c35c433849349dfd63dafad821b1 2024-11-27T11:35:07,559 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 57e760a789bb4a4791a85e07275bba1f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:07,559 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 6d8b7c5bba7742f0817809c5d3b5c267(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:07,559 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:07,559 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:07,561 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707307388; duration=0sec 2024-11-27T11:35:07,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:07,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:07,562 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707307394; duration=0sec 2024-11-27T11:35:07,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:07,563 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:07,563 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:07,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,581 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:07,581 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:07,581 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,582 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6705caa12e3f41c387922e628d8293c8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2465cba93a3f479e93073e61934525d4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/eaff47fd114e4f77bfe3c7ee676632e3] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.2 K 2024-11-27T11:35:07,583 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6705caa12e3f41c387922e628d8293c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732707305666 2024-11-27T11:35:07,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/a4a56ef44bd14d52b1c6ca47e2358e7d is 50, key is test_row_0/C:col10/1732707307388/Put/seqid=0 2024-11-27T11:35:07,584 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2465cba93a3f479e93073e61934525d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707305817 2024-11-27T11:35:07,585 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaff47fd114e4f77bfe3c7ee676632e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707306515 2024-11-27T11:35:07,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707367584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707367585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,589 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707367585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707367585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707367593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,607 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:07,608 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/7455744b98eb4ab1a5660fa6406d540b is 50, key is test_row_0/C:col10/1732707307179/Put/seqid=0 2024-11-27T11:35:07,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741850_1026 (size=12001) 2024-11-27T11:35:07,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741851_1027 (size=12104) 2024-11-27T11:35:07,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:07,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,723 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:07,764 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-27T11:35:07,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707367803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:07,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:07,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:07,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707367894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707367894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707367895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:07,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707367904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/a4a56ef44bd14d52b1c6ca47e2358e7d 2024-11-27T11:35:08,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:08,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:08,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:08,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/a7ef85be98204c5dad438c140c896a68 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/a7ef85be98204c5dad438c140c896a68 2024-11-27T11:35:08,044 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/7455744b98eb4ab1a5660fa6406d540b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7455744b98eb4ab1a5660fa6406d540b 2024-11-27T11:35:08,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/a7ef85be98204c5dad438c140c896a68, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T11:35:08,060 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 7455744b98eb4ab1a5660fa6406d540b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:08,060 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/6839c35c433849349dfd63dafad821b1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6839c35c433849349dfd63dafad821b1 2024-11-27T11:35:08,060 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707307395; duration=0sec 2024-11-27T11:35:08,060 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:08,060 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:08,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6839c35c433849349dfd63dafad821b1, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T11:35:08,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/a4a56ef44bd14d52b1c6ca47e2358e7d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/a4a56ef44bd14d52b1c6ca47e2358e7d 2024-11-27T11:35:08,088 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/a4a56ef44bd14d52b1c6ca47e2358e7d, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T11:35:08,090 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 080ce2238710db49165afb3a1a396a6b in 694ms, sequenceid=78, compaction requested=false 2024-11-27T11:35:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:08,116 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:08,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:08,128 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/f268aacc86584ea7a72d621e0edaa54c is 50, key is test_row_0/A:col10/1732707308113/Put/seqid=0 2024-11-27T11:35:08,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741852_1028 (size=14341) 2024-11-27T11:35:08,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/f268aacc86584ea7a72d621e0edaa54c 2024-11-27T11:35:08,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/08c93eca60fc4d169d9971d402feca8d is 50, key is test_row_0/B:col10/1732707308113/Put/seqid=0 2024-11-27T11:35:08,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:08,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:08,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:08,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741853_1029 (size=12001) 2024-11-27T11:35:08,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/08c93eca60fc4d169d9971d402feca8d 2024-11-27T11:35:08,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/fe78dd5902684afdac852d315800f38b is 50, key is test_row_0/C:col10/1732707308113/Put/seqid=0 2024-11-27T11:35:08,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741854_1030 (size=12001) 2024-11-27T11:35:08,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/fe78dd5902684afdac852d315800f38b 2024-11-27T11:35:08,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/f268aacc86584ea7a72d621e0edaa54c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/f268aacc86584ea7a72d621e0edaa54c 2024-11-27T11:35:08,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/f268aacc86584ea7a72d621e0edaa54c, entries=200, sequenceid=92, filesize=14.0 K 2024-11-27T11:35:08,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/08c93eca60fc4d169d9971d402feca8d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/08c93eca60fc4d169d9971d402feca8d 2024-11-27T11:35:08,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/08c93eca60fc4d169d9971d402feca8d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T11:35:08,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/fe78dd5902684afdac852d315800f38b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/fe78dd5902684afdac852d315800f38b 2024-11-27T11:35:08,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707368322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/fe78dd5902684afdac852d315800f38b, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T11:35:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,328 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 080ce2238710db49165afb3a1a396a6b in 213ms, sequenceid=92, compaction requested=true 2024-11-27T11:35:08,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:08,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:08,330 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:08,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:08,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:08,330 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:08,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:08,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:08,335 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:08,335 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:08,335 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:08,335 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:08,335 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,335 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,335 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6d8b7c5bba7742f0817809c5d3b5c267, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6839c35c433849349dfd63dafad821b1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/08c93eca60fc4d169d9971d402feca8d] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.3 K 2024-11-27T11:35:08,335 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/57e760a789bb4a4791a85e07275bba1f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/a7ef85be98204c5dad438c140c896a68, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/f268aacc86584ea7a72d621e0edaa54c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.5 K 2024-11-27T11:35:08,336 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e760a789bb4a4791a85e07275bba1f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707306515 2024-11-27T11:35:08,336 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d8b7c5bba7742f0817809c5d3b5c267, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707306515 2024-11-27T11:35:08,337 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6839c35c433849349dfd63dafad821b1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732707307252 2024-11-27T11:35:08,337 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7ef85be98204c5dad438c140c896a68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732707307252 2024-11-27T11:35:08,339 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 08c93eca60fc4d169d9971d402feca8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707307447 2024-11-27T11:35:08,339 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f268aacc86584ea7a72d621e0edaa54c, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707307447 2024-11-27T11:35:08,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-27T11:35:08,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,353 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:35:08,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/c258c37cf5b049d4942a25434350f682 is 50, key is test_row_0/A:col10/1732707308277/Put/seqid=0 2024-11-27T11:35:08,367 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#19 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:08,371 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#20 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:08,372 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/13ec9a9bad0548fbb010dc979a6be414 is 50, key is test_row_0/A:col10/1732707308113/Put/seqid=0 2024-11-27T11:35:08,375 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a8263c7edcad4d3fbed077d8c1bc94b9 is 50, key is test_row_0/B:col10/1732707308113/Put/seqid=0 2024-11-27T11:35:08,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741855_1031 (size=12001) 2024-11-27T11:35:08,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741856_1032 (size=12207) 2024-11-27T11:35:08,405 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:35:08,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741857_1033 (size=12207) 2024-11-27T11:35:08,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:08,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:08,411 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/13ec9a9bad0548fbb010dc979a6be414 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/13ec9a9bad0548fbb010dc979a6be414 2024-11-27T11:35:08,424 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 13ec9a9bad0548fbb010dc979a6be414(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:08,425 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,425 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707308329; duration=0sec 2024-11-27T11:35:08,425 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:08,425 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:08,425 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:08,428 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:08,428 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:08,429 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,429 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7455744b98eb4ab1a5660fa6406d540b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/a4a56ef44bd14d52b1c6ca47e2358e7d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/fe78dd5902684afdac852d315800f38b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.3 K 2024-11-27T11:35:08,430 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7455744b98eb4ab1a5660fa6406d540b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707306515 2024-11-27T11:35:08,431 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a56ef44bd14d52b1c6ca47e2358e7d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732707307252 2024-11-27T11:35:08,432 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe78dd5902684afdac852d315800f38b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707307447 2024-11-27T11:35:08,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707368423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707368432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707368432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707368434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707368438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,475 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:08,476 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/270cd73c51f04ad989c5d6c7dd3e28c1 is 50, key is test_row_0/C:col10/1732707308113/Put/seqid=0 2024-11-27T11:35:08,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741858_1034 (size=12207) 2024-11-27T11:35:08,525 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/270cd73c51f04ad989c5d6c7dd3e28c1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/270cd73c51f04ad989c5d6c7dd3e28c1 2024-11-27T11:35:08,540 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 270cd73c51f04ad989c5d6c7dd3e28c1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:08,540 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,540 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707308330; duration=0sec 2024-11-27T11:35:08,541 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:08,541 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:08,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707368554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707368554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707368554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707368653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,705 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T11:35:08,706 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-27T11:35:08,708 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T11:35:08,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-27T11:35:08,709 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-27T11:35:08,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-27T11:35:08,711 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T11:35:08,711 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T11:35:08,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707368763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707368773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,778 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707368773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:08,780 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/c258c37cf5b049d4942a25434350f682 2024-11-27T11:35:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:08,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/55c4f8ef3f9b46829227467381f0286f is 50, key is test_row_0/B:col10/1732707308277/Put/seqid=0 2024-11-27T11:35:08,824 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a8263c7edcad4d3fbed077d8c1bc94b9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a8263c7edcad4d3fbed077d8c1bc94b9 2024-11-27T11:35:08,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741859_1035 (size=12001) 2024-11-27T11:35:08,831 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/55c4f8ef3f9b46829227467381f0286f 2024-11-27T11:35:08,843 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into a8263c7edcad4d3fbed077d8c1bc94b9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:08,843 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,843 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707308330; duration=0sec 2024-11-27T11:35:08,843 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:08,843 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:08,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/17da9b666b84417c90263eba2c4d179a is 50, key is test_row_0/C:col10/1732707308277/Put/seqid=0 2024-11-27T11:35:08,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741860_1036 (size=12001) 2024-11-27T11:35:08,871 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/17da9b666b84417c90263eba2c4d179a 2024-11-27T11:35:08,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/c258c37cf5b049d4942a25434350f682 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/c258c37cf5b049d4942a25434350f682 2024-11-27T11:35:08,897 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/c258c37cf5b049d4942a25434350f682, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T11:35:08,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/55c4f8ef3f9b46829227467381f0286f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/55c4f8ef3f9b46829227467381f0286f 2024-11-27T11:35:08,916 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/55c4f8ef3f9b46829227467381f0286f, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T11:35:08,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/17da9b666b84417c90263eba2c4d179a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/17da9b666b84417c90263eba2c4d179a 2024-11-27T11:35:08,941 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/17da9b666b84417c90263eba2c4d179a, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T11:35:08,944 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 080ce2238710db49165afb3a1a396a6b in 590ms, sequenceid=117, compaction requested=false 2024-11-27T11:35:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:08,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-27T11:35:08,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-27T11:35:08,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-27T11:35:08,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3300 sec 2024-11-27T11:35:08,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.3540 sec 2024-11-27T11:35:08,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:08,972 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:08,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:35:08,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bba5f8b0d54b41f688b344f8de9e3387 is 50, key is test_row_0/A:col10/1732707308417/Put/seqid=0 2024-11-27T11:35:09,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741861_1037 (size=14441) 2024-11-27T11:35:09,016 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bba5f8b0d54b41f688b344f8de9e3387 2024-11-27T11:35:09,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/df9253a1d5c645c6a89829df0857b535 is 50, key is test_row_0/B:col10/1732707308417/Put/seqid=0 2024-11-27T11:35:09,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741862_1038 (size=12101) 2024-11-27T11:35:09,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707369107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707369109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707369110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707369111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,118 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707369222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707369223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707369223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707369223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,227 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707369433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707369433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707369434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707369434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707369443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,454 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in 080ce2238710db49165afb3a1a396a6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/df9253a1d5c645c6a89829df0857b535 2024-11-27T11:35:09,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/1acaa3fe7ae445328be87f86a32377fb is 50, key is test_row_0/C:col10/1732707308417/Put/seqid=0 2024-11-27T11:35:09,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741863_1039 (size=12101) 2024-11-27T11:35:09,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/1acaa3fe7ae445328be87f86a32377fb 2024-11-27T11:35:09,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bba5f8b0d54b41f688b344f8de9e3387 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bba5f8b0d54b41f688b344f8de9e3387 2024-11-27T11:35:09,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bba5f8b0d54b41f688b344f8de9e3387, entries=200, sequenceid=133, filesize=14.1 K 2024-11-27T11:35:09,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/df9253a1d5c645c6a89829df0857b535 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/df9253a1d5c645c6a89829df0857b535 2024-11-27T11:35:09,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/df9253a1d5c645c6a89829df0857b535, entries=150, sequenceid=133, filesize=11.8 K 2024-11-27T11:35:09,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/1acaa3fe7ae445328be87f86a32377fb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/1acaa3fe7ae445328be87f86a32377fb 2024-11-27T11:35:09,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/1acaa3fe7ae445328be87f86a32377fb, entries=150, sequenceid=133, filesize=11.8 K 2024-11-27T11:35:09,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 080ce2238710db49165afb3a1a396a6b in 617ms, sequenceid=133, compaction requested=true 2024-11-27T11:35:09,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:09,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:09,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:09,590 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:09,590 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:09,592 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:09,592 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:09,592 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:09,593 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a8263c7edcad4d3fbed077d8c1bc94b9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/55c4f8ef3f9b46829227467381f0286f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/df9253a1d5c645c6a89829df0857b535] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.5 K 2024-11-27T11:35:09,593 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:09,593 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:09,593 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:09,593 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/13ec9a9bad0548fbb010dc979a6be414, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/c258c37cf5b049d4942a25434350f682, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bba5f8b0d54b41f688b344f8de9e3387] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.7 K 2024-11-27T11:35:09,594 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a8263c7edcad4d3fbed077d8c1bc94b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707307447 2024-11-27T11:35:09,595 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13ec9a9bad0548fbb010dc979a6be414, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707307447 2024-11-27T11:35:09,595 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 55c4f8ef3f9b46829227467381f0286f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732707308264 2024-11-27T11:35:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:09,595 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c258c37cf5b049d4942a25434350f682, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732707308264 2024-11-27T11:35:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:09,596 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting df9253a1d5c645c6a89829df0857b535, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732707308417 2024-11-27T11:35:09,596 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bba5f8b0d54b41f688b344f8de9e3387, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732707308417 2024-11-27T11:35:09,614 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#27 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:09,615 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/277631b0e315411db50f599e4688563a is 50, key is test_row_0/A:col10/1732707308417/Put/seqid=0 2024-11-27T11:35:09,623 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:09,624 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/cc8bcc9017b04964b911184508f9a2c9 is 50, key is test_row_0/B:col10/1732707308417/Put/seqid=0 2024-11-27T11:35:09,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741864_1040 (size=12409) 2024-11-27T11:35:09,662 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/277631b0e315411db50f599e4688563a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/277631b0e315411db50f599e4688563a 2024-11-27T11:35:09,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741865_1041 (size=12409) 2024-11-27T11:35:09,677 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 277631b0e315411db50f599e4688563a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:09,677 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:09,677 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707309589; duration=0sec 2024-11-27T11:35:09,677 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:09,677 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:09,678 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:09,681 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:09,681 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/cc8bcc9017b04964b911184508f9a2c9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cc8bcc9017b04964b911184508f9a2c9 2024-11-27T11:35:09,681 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:09,681 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:09,681 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/270cd73c51f04ad989c5d6c7dd3e28c1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/17da9b666b84417c90263eba2c4d179a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/1acaa3fe7ae445328be87f86a32377fb] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.5 K 2024-11-27T11:35:09,682 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 270cd73c51f04ad989c5d6c7dd3e28c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707307447 2024-11-27T11:35:09,683 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17da9b666b84417c90263eba2c4d179a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732707308264 2024-11-27T11:35:09,683 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1acaa3fe7ae445328be87f86a32377fb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732707308417 2024-11-27T11:35:09,691 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into cc8bcc9017b04964b911184508f9a2c9(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:09,691 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:09,691 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707309590; duration=0sec 2024-11-27T11:35:09,692 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:09,692 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:09,701 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#29 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:09,702 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/019ea283a1e34bc48879dc6d87467288 is 50, key is test_row_0/C:col10/1732707308417/Put/seqid=0 2024-11-27T11:35:09,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741866_1042 (size=12409) 2024-11-27T11:35:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:09,747 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:09,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:35:09,753 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/019ea283a1e34bc48879dc6d87467288 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/019ea283a1e34bc48879dc6d87467288 2024-11-27T11:35:09,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6ece38beeeec46d7810c8983c1a154b0 is 50, key is test_row_0/A:col10/1732707309745/Put/seqid=0 2024-11-27T11:35:09,769 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 019ea283a1e34bc48879dc6d87467288(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:09,769 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:09,769 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707309595; duration=0sec 2024-11-27T11:35:09,769 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:09,769 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:09,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707369763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707369768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707369770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707369773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-27T11:35:09,784 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:09,784 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:09,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741867_1043 (size=14541) 2024-11-27T11:35:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-27T11:35:09,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6ece38beeeec46d7810c8983c1a154b0 2024-11-27T11:35:09,788 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:09,790 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:09,790 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:09,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/ecd7e0f03b5d48a79ed479bc11c23b3d is 50, key is test_row_0/B:col10/1732707309745/Put/seqid=0 2024-11-27T11:35:09,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741868_1044 (size=12151) 2024-11-27T11:35:09,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/ecd7e0f03b5d48a79ed479bc11c23b3d 2024-11-27T11:35:09,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f1b9f029be364fc9abf2d204fea4db0a is 50, key is test_row_0/C:col10/1732707309745/Put/seqid=0 2024-11-27T11:35:09,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741869_1045 (size=12151) 2024-11-27T11:35:09,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707369873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707369882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707369883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707369883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:09,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:09,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:09,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:09,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707370083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707370093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:10,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707370102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707370103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f1b9f029be364fc9abf2d204fea4db0a 2024-11-27T11:35:10,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6ece38beeeec46d7810c8983c1a154b0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6ece38beeeec46d7810c8983c1a154b0 2024-11-27T11:35:10,289 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6ece38beeeec46d7810c8983c1a154b0, entries=200, sequenceid=161, filesize=14.2 K 2024-11-27T11:35:10,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/ecd7e0f03b5d48a79ed479bc11c23b3d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ecd7e0f03b5d48a79ed479bc11c23b3d 2024-11-27T11:35:10,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ecd7e0f03b5d48a79ed479bc11c23b3d, entries=150, sequenceid=161, filesize=11.9 K 2024-11-27T11:35:10,303 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f1b9f029be364fc9abf2d204fea4db0a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f1b9f029be364fc9abf2d204fea4db0a 2024-11-27T11:35:10,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f1b9f029be364fc9abf2d204fea4db0a, entries=150, sequenceid=161, filesize=11.9 K 2024-11-27T11:35:10,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 080ce2238710db49165afb3a1a396a6b in 569ms, sequenceid=161, compaction requested=false 2024-11-27T11:35:10,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:10,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:10,406 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:10,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:35:10,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:10,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:10,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:10,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/08859fb86c9245b3ad495d3eeefbf882 is 50, key is test_row_0/A:col10/1732707309766/Put/seqid=0 2024-11-27T11:35:10,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741870_1046 (size=14541) 2024-11-27T11:35:10,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707370456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707370458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707370461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707370469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707370573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707370573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707370585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707370585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:10,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707370782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707370783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,788 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707370793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707370793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:10,795 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:10,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/08859fb86c9245b3ad495d3eeefbf882 2024-11-27T11:35:10,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/0bb381a8d4c7488192b3ada902accc6b is 50, key is test_row_0/B:col10/1732707309766/Put/seqid=0 2024-11-27T11:35:10,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:10,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:10,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:10,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741871_1047 (size=12151) 2024-11-27T11:35:10,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/0bb381a8d4c7488192b3ada902accc6b 2024-11-27T11:35:10,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2da3c5ed04cd46b686e040ce46e33b97 is 50, key is test_row_0/C:col10/1732707309766/Put/seqid=0 2024-11-27T11:35:10,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:10,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741872_1048 (size=12151) 2024-11-27T11:35:11,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,094 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707371093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,096 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707371093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707371103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707371103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,108 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2da3c5ed04cd46b686e040ce46e33b97 2024-11-27T11:35:11,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/08859fb86c9245b3ad495d3eeefbf882 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/08859fb86c9245b3ad495d3eeefbf882 2024-11-27T11:35:11,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/08859fb86c9245b3ad495d3eeefbf882, entries=200, sequenceid=174, filesize=14.2 K 2024-11-27T11:35:11,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/0bb381a8d4c7488192b3ada902accc6b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0bb381a8d4c7488192b3ada902accc6b 2024-11-27T11:35:11,374 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0bb381a8d4c7488192b3ada902accc6b, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T11:35:11,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2da3c5ed04cd46b686e040ce46e33b97 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2da3c5ed04cd46b686e040ce46e33b97 2024-11-27T11:35:11,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2da3c5ed04cd46b686e040ce46e33b97, entries=150, sequenceid=174, filesize=11.9 K 2024-11-27T11:35:11,382 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 080ce2238710db49165afb3a1a396a6b in 976ms, sequenceid=174, compaction requested=true 2024-11-27T11:35:11,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:11,383 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:11,383 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:11,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:11,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:11,384 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41491 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:11,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:11,384 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:11,384 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:11,384 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,385 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:11,385 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,385 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/277631b0e315411db50f599e4688563a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6ece38beeeec46d7810c8983c1a154b0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/08859fb86c9245b3ad495d3eeefbf882] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=40.5 K 2024-11-27T11:35:11,385 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cc8bcc9017b04964b911184508f9a2c9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ecd7e0f03b5d48a79ed479bc11c23b3d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0bb381a8d4c7488192b3ada902accc6b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.9 K 2024-11-27T11:35:11,385 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 277631b0e315411db50f599e4688563a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732707308417 2024-11-27T11:35:11,385 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting cc8bcc9017b04964b911184508f9a2c9, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732707308417 2024-11-27T11:35:11,386 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ece38beeeec46d7810c8983c1a154b0, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732707309092 2024-11-27T11:35:11,386 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd7e0f03b5d48a79ed479bc11c23b3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732707309092 2024-11-27T11:35:11,386 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08859fb86c9245b3ad495d3eeefbf882, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732707309765 2024-11-27T11:35:11,387 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb381a8d4c7488192b3ada902accc6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732707309766 2024-11-27T11:35:11,406 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#36 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:11,407 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/91bf66c703114bf798d6e85b2334fe91 is 50, key is test_row_0/B:col10/1732707309766/Put/seqid=0 2024-11-27T11:35:11,411 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#37 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:11,412 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6e54141c548746be81616edfd4e32c3f is 50, key is test_row_0/A:col10/1732707309766/Put/seqid=0 2024-11-27T11:35:11,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741873_1049 (size=12561) 2024-11-27T11:35:11,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741874_1050 (size=12561) 2024-11-27T11:35:11,442 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/91bf66c703114bf798d6e85b2334fe91 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/91bf66c703114bf798d6e85b2334fe91 2024-11-27T11:35:11,453 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 91bf66c703114bf798d6e85b2334fe91(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:11,454 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:11,454 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707311383; duration=0sec 2024-11-27T11:35:11,454 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:11,454 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:11,454 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:11,456 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:11,456 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:11,456 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,456 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/019ea283a1e34bc48879dc6d87467288, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f1b9f029be364fc9abf2d204fea4db0a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2da3c5ed04cd46b686e040ce46e33b97] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=35.9 K 2024-11-27T11:35:11,457 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 019ea283a1e34bc48879dc6d87467288, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732707308417 2024-11-27T11:35:11,458 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f1b9f029be364fc9abf2d204fea4db0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732707309092 2024-11-27T11:35:11,459 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da3c5ed04cd46b686e040ce46e33b97, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732707309766 2024-11-27T11:35:11,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:11,479 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:11,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:35:11,492 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#38 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:11,493 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/e8f8c1e6b4334777911fffe06dbafa02 is 50, key is test_row_0/C:col10/1732707309766/Put/seqid=0 2024-11-27T11:35:11,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8003e345c5c147f680b6304984c67064 is 50, key is test_row_0/A:col10/1732707310451/Put/seqid=0 2024-11-27T11:35:11,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741875_1051 (size=12561) 2024-11-27T11:35:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707371527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,532 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/e8f8c1e6b4334777911fffe06dbafa02 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e8f8c1e6b4334777911fffe06dbafa02 2024-11-27T11:35:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,544 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into e8f8c1e6b4334777911fffe06dbafa02(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:11,544 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:11,544 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707311384; duration=0sec 2024-11-27T11:35:11,544 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:11,544 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:11,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741876_1052 (size=14541) 2024-11-27T11:35:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707371602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707371602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707371613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707371613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707371642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,825 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,847 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6e54141c548746be81616edfd4e32c3f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6e54141c548746be81616edfd4e32c3f 2024-11-27T11:35:11,856 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 6e54141c548746be81616edfd4e32c3f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:11,856 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:11,856 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707311382; duration=0sec 2024-11-27T11:35:11,856 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:11,856 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:11,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:11,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707371853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:11,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:11,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:11,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8003e345c5c147f680b6304984c67064 2024-11-27T11:35:11,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/fed9ca3c35ba4e01bc322388b052d9e6 is 50, key is test_row_0/B:col10/1732707310451/Put/seqid=0 2024-11-27T11:35:11,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741877_1053 (size=12151) 2024-11-27T11:35:11,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/fed9ca3c35ba4e01bc322388b052d9e6 2024-11-27T11:35:11,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:11,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:11,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:11,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:11,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2865e16edfc94b84a893a45b966458d8 is 50, key is test_row_0/C:col10/1732707310451/Put/seqid=0 2024-11-27T11:35:12,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741878_1054 (size=12151) 2024-11-27T11:35:12,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=200 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2865e16edfc94b84a893a45b966458d8 2024-11-27T11:35:12,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8003e345c5c147f680b6304984c67064 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8003e345c5c147f680b6304984c67064 2024-11-27T11:35:12,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8003e345c5c147f680b6304984c67064, entries=200, sequenceid=200, filesize=14.2 K 2024-11-27T11:35:12,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/fed9ca3c35ba4e01bc322388b052d9e6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fed9ca3c35ba4e01bc322388b052d9e6 2024-11-27T11:35:12,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fed9ca3c35ba4e01bc322388b052d9e6, entries=150, sequenceid=200, filesize=11.9 K 2024-11-27T11:35:12,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/2865e16edfc94b84a893a45b966458d8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2865e16edfc94b84a893a45b966458d8 2024-11-27T11:35:12,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2865e16edfc94b84a893a45b966458d8, entries=150, sequenceid=200, filesize=11.9 K 2024-11-27T11:35:12,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 080ce2238710db49165afb3a1a396a6b in 582ms, sequenceid=200, compaction requested=false 2024-11-27T11:35:12,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:12,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-27T11:35:12,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:12,138 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/77f1b9eb94024e76a16ed7331d0d6750 is 50, key is test_row_0/A:col10/1732707311523/Put/seqid=0 2024-11-27T11:35:12,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741879_1055 (size=12151) 2024-11-27T11:35:12,159 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/77f1b9eb94024e76a16ed7331d0d6750 2024-11-27T11:35:12,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:12,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:12,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5a6e85fc31f34456b941cb0f10eef429 is 50, key is test_row_0/B:col10/1732707311523/Put/seqid=0 2024-11-27T11:35:12,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741880_1056 (size=12151) 2024-11-27T11:35:12,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707372274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707372382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,592 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5a6e85fc31f34456b941cb0f10eef429 2024-11-27T11:35:12,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707372593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/c851fca7a6b44baaa1a00b2ab1a72334 is 50, key is test_row_0/C:col10/1732707311523/Put/seqid=0 2024-11-27T11:35:12,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741881_1057 (size=12151) 2024-11-27T11:35:12,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707372612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707372613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,616 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in 080ce2238710db49165afb3a1a396a6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,617 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in 080ce2238710db49165afb3a1a396a6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707372623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,625 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in 080ce2238710db49165afb3a1a396a6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707372632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,634 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 080ce2238710db49165afb3a1a396a6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:12,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:12,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707372902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:12,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:13,012 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/c851fca7a6b44baaa1a00b2ab1a72334 2024-11-27T11:35:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/77f1b9eb94024e76a16ed7331d0d6750 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/77f1b9eb94024e76a16ed7331d0d6750 2024-11-27T11:35:13,026 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/77f1b9eb94024e76a16ed7331d0d6750, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T11:35:13,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5a6e85fc31f34456b941cb0f10eef429 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5a6e85fc31f34456b941cb0f10eef429 2024-11-27T11:35:13,034 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5a6e85fc31f34456b941cb0f10eef429, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T11:35:13,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/c851fca7a6b44baaa1a00b2ab1a72334 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/c851fca7a6b44baaa1a00b2ab1a72334 2024-11-27T11:35:13,042 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/c851fca7a6b44baaa1a00b2ab1a72334, entries=150, sequenceid=213, filesize=11.9 K 2024-11-27T11:35:13,044 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 080ce2238710db49165afb3a1a396a6b in 907ms, sequenceid=213, compaction requested=true 2024-11-27T11:35:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:13,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-27T11:35:13,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-27T11:35:13,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-27T11:35:13,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2580 sec 2024-11-27T11:35:13,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 3.2680 sec 2024-11-27T11:35:13,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:13,415 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:13,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:35:13,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/7e9ac9b5232e41bb887e93ed6f664c32 is 50, key is test_row_0/A:col10/1732707312265/Put/seqid=0 2024-11-27T11:35:13,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741882_1058 (size=14541) 2024-11-27T11:35:13,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:13,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707373451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:13,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:13,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:13,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:13,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707373562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:13,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:13,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:13,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707373773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:13,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/7e9ac9b5232e41bb887e93ed6f664c32 2024-11-27T11:35:13,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d3345ca27e424d94b81fa4cbb7eb155b is 50, key is test_row_0/B:col10/1732707312265/Put/seqid=0 2024-11-27T11:35:13,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741883_1059 (size=12151) 2024-11-27T11:35:13,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-27T11:35:13,963 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:13,963 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:13,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-27T11:35:13,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T11:35:13,966 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:13,967 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:13,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:14,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T11:35:14,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707374083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,086 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T11:35:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:14,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:14,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:14,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d3345ca27e424d94b81fa4cbb7eb155b 2024-11-27T11:35:14,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/4b32645fbde047748112d37fd40bc686 is 50, key is test_row_0/C:col10/1732707312265/Put/seqid=0 2024-11-27T11:35:14,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741884_1060 (size=12151) 2024-11-27T11:35:14,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/4b32645fbde047748112d37fd40bc686 2024-11-27T11:35:14,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/7e9ac9b5232e41bb887e93ed6f664c32 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/7e9ac9b5232e41bb887e93ed6f664c32 2024-11-27T11:35:14,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T11:35:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:14,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:14,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/7e9ac9b5232e41bb887e93ed6f664c32, entries=200, sequenceid=239, filesize=14.2 K 2024-11-27T11:35:14,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d3345ca27e424d94b81fa4cbb7eb155b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d3345ca27e424d94b81fa4cbb7eb155b 2024-11-27T11:35:14,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T11:35:14,285 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d3345ca27e424d94b81fa4cbb7eb155b, entries=150, sequenceid=239, filesize=11.9 K 2024-11-27T11:35:14,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/4b32645fbde047748112d37fd40bc686 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/4b32645fbde047748112d37fd40bc686 2024-11-27T11:35:14,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/4b32645fbde047748112d37fd40bc686, entries=150, sequenceid=239, filesize=11.9 K 2024-11-27T11:35:14,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 080ce2238710db49165afb3a1a396a6b in 877ms, sequenceid=239, compaction requested=true 2024-11-27T11:35:14,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:14,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:14,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:14,292 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:14,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:14,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:14,293 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:14,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:14,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:14,294 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:14,294 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:14,294 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:14,294 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:14,295 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,295 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,295 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/91bf66c703114bf798d6e85b2334fe91, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fed9ca3c35ba4e01bc322388b052d9e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5a6e85fc31f34456b941cb0f10eef429, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d3345ca27e424d94b81fa4cbb7eb155b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=47.9 K 2024-11-27T11:35:14,295 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6e54141c548746be81616edfd4e32c3f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8003e345c5c147f680b6304984c67064, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/77f1b9eb94024e76a16ed7331d0d6750, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/7e9ac9b5232e41bb887e93ed6f664c32] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=52.5 K 2024-11-27T11:35:14,295 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 91bf66c703114bf798d6e85b2334fe91, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732707309766 2024-11-27T11:35:14,295 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e54141c548746be81616edfd4e32c3f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732707309766 2024-11-27T11:35:14,296 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting fed9ca3c35ba4e01bc322388b052d9e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732707310451 2024-11-27T11:35:14,296 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8003e345c5c147f680b6304984c67064, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732707310451 2024-11-27T11:35:14,296 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a6e85fc31f34456b941cb0f10eef429, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732707311487 2024-11-27T11:35:14,296 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77f1b9eb94024e76a16ed7331d0d6750, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732707311487 2024-11-27T11:35:14,297 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9ac9b5232e41bb887e93ed6f664c32, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732707312259 2024-11-27T11:35:14,297 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d3345ca27e424d94b81fa4cbb7eb155b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732707312265 2024-11-27T11:35:14,311 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:14,312 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/ea59a0f714c543a4877b7744b49106d6 is 50, key is test_row_0/B:col10/1732707312265/Put/seqid=0 2024-11-27T11:35:14,319 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:14,320 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/1c05fbad67f94d3ebb2ce8b54b0390a4 is 50, key is test_row_0/A:col10/1732707312265/Put/seqid=0 2024-11-27T11:35:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741885_1061 (size=12697) 2024-11-27T11:35:14,336 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/ea59a0f714c543a4877b7744b49106d6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ea59a0f714c543a4877b7744b49106d6 2024-11-27T11:35:14,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741886_1062 (size=12697) 2024-11-27T11:35:14,347 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into ea59a0f714c543a4877b7744b49106d6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:14,347 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:14,347 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=12, startTime=1732707314292; duration=0sec 2024-11-27T11:35:14,347 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:14,347 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:14,347 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:14,351 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:14,351 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:14,352 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,352 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e8f8c1e6b4334777911fffe06dbafa02, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2865e16edfc94b84a893a45b966458d8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/c851fca7a6b44baaa1a00b2ab1a72334, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/4b32645fbde047748112d37fd40bc686] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=47.9 K 2024-11-27T11:35:14,352 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/1c05fbad67f94d3ebb2ce8b54b0390a4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/1c05fbad67f94d3ebb2ce8b54b0390a4 2024-11-27T11:35:14,352 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f8c1e6b4334777911fffe06dbafa02, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732707309766 2024-11-27T11:35:14,353 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2865e16edfc94b84a893a45b966458d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=200, earliestPutTs=1732707310451 2024-11-27T11:35:14,354 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c851fca7a6b44baaa1a00b2ab1a72334, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732707311487 2024-11-27T11:35:14,354 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b32645fbde047748112d37fd40bc686, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732707312265 2024-11-27T11:35:14,362 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 1c05fbad67f94d3ebb2ce8b54b0390a4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:14,362 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:14,362 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=12, startTime=1732707314292; duration=0sec 2024-11-27T11:35:14,363 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:14,363 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:14,367 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#50 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:14,368 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/004997f048964f8e899c92bbb7e20c60 is 50, key is test_row_0/C:col10/1732707312265/Put/seqid=0 2024-11-27T11:35:14,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741887_1063 (size=12697) 2024-11-27T11:35:14,398 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/004997f048964f8e899c92bbb7e20c60 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/004997f048964f8e899c92bbb7e20c60 2024-11-27T11:35:14,405 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 004997f048964f8e899c92bbb7e20c60(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:14,405 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:14,405 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=12, startTime=1732707314293; duration=0sec 2024-11-27T11:35:14,406 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:14,406 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:14,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-27T11:35:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:14,430 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T11:35:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/42bf1c84d3854e789c36bcdb8a98cec2 is 50, key is test_row_0/A:col10/1732707313437/Put/seqid=0 2024-11-27T11:35:14,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741888_1064 (size=12151) 2024-11-27T11:35:14,459 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/42bf1c84d3854e789c36bcdb8a98cec2 2024-11-27T11:35:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/360f1c912db0456b80c23ce6e564ade2 is 50, key is test_row_0/B:col10/1732707313437/Put/seqid=0 2024-11-27T11:35:14,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741889_1065 (size=12151) 2024-11-27T11:35:14,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T11:35:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:14,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707374651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707374651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707374652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707374654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707374654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707374762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707374762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,765 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707374762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707374762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707374763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,879 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/360f1c912db0456b80c23ce6e564ade2 2024-11-27T11:35:14,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/b719c2cbd1ac4c1eb5474a80a756bee2 is 50, key is test_row_0/C:col10/1732707313437/Put/seqid=0 2024-11-27T11:35:14,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741890_1066 (size=12151) 2024-11-27T11:35:14,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707374972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707374973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707374973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,975 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707374973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:14,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707374973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:14,978 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T11:35:15,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707375283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707375283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707375283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707375283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707375283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,303 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/b719c2cbd1ac4c1eb5474a80a756bee2 2024-11-27T11:35:15,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/42bf1c84d3854e789c36bcdb8a98cec2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/42bf1c84d3854e789c36bcdb8a98cec2 2024-11-27T11:35:15,323 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/42bf1c84d3854e789c36bcdb8a98cec2, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T11:35:15,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/360f1c912db0456b80c23ce6e564ade2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/360f1c912db0456b80c23ce6e564ade2 2024-11-27T11:35:15,332 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/360f1c912db0456b80c23ce6e564ade2, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T11:35:15,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/b719c2cbd1ac4c1eb5474a80a756bee2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/b719c2cbd1ac4c1eb5474a80a756bee2 2024-11-27T11:35:15,339 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/b719c2cbd1ac4c1eb5474a80a756bee2, entries=150, sequenceid=252, filesize=11.9 K 2024-11-27T11:35:15,342 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 080ce2238710db49165afb3a1a396a6b in 912ms, sequenceid=252, compaction requested=false 2024-11-27T11:35:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:15,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-27T11:35:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-27T11:35:15,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-27T11:35:15,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3780 sec 2024-11-27T11:35:15,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3870 sec 2024-11-27T11:35:15,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:15,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:15,796 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-27T11:35:15,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3bac3cb8c57348dea9a8c1a3c2c6faf3 is 50, key is test_row_0/A:col10/1732707314649/Put/seqid=0 2024-11-27T11:35:15,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741891_1067 (size=17181) 2024-11-27T11:35:15,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3bac3cb8c57348dea9a8c1a3c2c6faf3 2024-11-27T11:35:15,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707375806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707375806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707375807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707375808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707375809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a22078f15c4c4094b63dac5cbb025c01 is 50, key is test_row_0/B:col10/1732707314649/Put/seqid=0 2024-11-27T11:35:15,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741892_1068 (size=12301) 2024-11-27T11:35:15,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707375922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707375922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707375923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707375923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707375923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:15,927 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-27T11:35:16,113 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:16,113 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:16,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-27T11:35:16,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T11:35:16,116 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:16,116 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:16,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:16,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707376133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707376133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707376133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707376133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707376133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T11:35:16,237 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a22078f15c4c4094b63dac5cbb025c01 2024-11-27T11:35:16,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/908d041f31da4dfa89db5bd0447df1cd is 50, key is test_row_0/C:col10/1732707314649/Put/seqid=0 2024-11-27T11:35:16,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741893_1069 (size=12301) 2024-11-27T11:35:16,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T11:35:16,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:16,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T11:35:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:16,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T11:35:16,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707376442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707376442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707376443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707376443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707376443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:16,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:16,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T11:35:16,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:16,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:16,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/908d041f31da4dfa89db5bd0447df1cd 2024-11-27T11:35:16,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3bac3cb8c57348dea9a8c1a3c2c6faf3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3bac3cb8c57348dea9a8c1a3c2c6faf3 2024-11-27T11:35:16,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3bac3cb8c57348dea9a8c1a3c2c6faf3, entries=250, sequenceid=281, filesize=16.8 K 2024-11-27T11:35:16,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/a22078f15c4c4094b63dac5cbb025c01 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a22078f15c4c4094b63dac5cbb025c01 2024-11-27T11:35:16,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a22078f15c4c4094b63dac5cbb025c01, entries=150, sequenceid=281, filesize=12.0 K 2024-11-27T11:35:16,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/908d041f31da4dfa89db5bd0447df1cd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/908d041f31da4dfa89db5bd0447df1cd 2024-11-27T11:35:16,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/908d041f31da4dfa89db5bd0447df1cd, entries=150, sequenceid=281, filesize=12.0 K 2024-11-27T11:35:16,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 080ce2238710db49165afb3a1a396a6b in 891ms, sequenceid=281, compaction requested=true 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:16,687 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:16,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T11:35:16,687 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:16,689 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:16,689 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:16,689 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,689 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/1c05fbad67f94d3ebb2ce8b54b0390a4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/42bf1c84d3854e789c36bcdb8a98cec2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3bac3cb8c57348dea9a8c1a3c2c6faf3] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=41.0 K 2024-11-27T11:35:16,689 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:16,690 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:16,690 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,690 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ea59a0f714c543a4877b7744b49106d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/360f1c912db0456b80c23ce6e564ade2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a22078f15c4c4094b63dac5cbb025c01] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=36.3 K 2024-11-27T11:35:16,690 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c05fbad67f94d3ebb2ce8b54b0390a4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732707312265 2024-11-27T11:35:16,691 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ea59a0f714c543a4877b7744b49106d6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732707312265 2024-11-27T11:35:16,691 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 360f1c912db0456b80c23ce6e564ade2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732707313437 2024-11-27T11:35:16,691 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42bf1c84d3854e789c36bcdb8a98cec2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732707313437 2024-11-27T11:35:16,692 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a22078f15c4c4094b63dac5cbb025c01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732707314649 2024-11-27T11:35:16,692 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bac3cb8c57348dea9a8c1a3c2c6faf3, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732707314649 2024-11-27T11:35:16,704 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:16,705 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/9ee8f5dcac144db283bbd9b6766c05ea is 50, key is test_row_0/A:col10/1732707314649/Put/seqid=0 2024-11-27T11:35:16,706 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#58 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:16,707 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/37aa0b6b6c0841ad8ca8bee09deae2eb is 50, key is test_row_0/B:col10/1732707314649/Put/seqid=0 2024-11-27T11:35:16,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741894_1070 (size=12949) 2024-11-27T11:35:16,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741895_1071 (size=12949) 2024-11-27T11:35:16,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-27T11:35:16,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,733 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T11:35:16,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bdc7522936354df8828230dec3d36b87 is 50, key is test_row_0/A:col10/1732707315801/Put/seqid=0 2024-11-27T11:35:16,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T11:35:16,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741896_1072 (size=12301) 2024-11-27T11:35:16,746 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bdc7522936354df8828230dec3d36b87 2024-11-27T11:35:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d72c8fd2ba8549a784cfecc0cdcf35d9 is 50, key is test_row_0/B:col10/1732707315801/Put/seqid=0 2024-11-27T11:35:16,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741897_1073 (size=12301) 2024-11-27T11:35:16,768 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d72c8fd2ba8549a784cfecc0cdcf35d9 2024-11-27T11:35:16,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/201ac72000a346fb9a6b075976f85728 is 50, key is test_row_0/C:col10/1732707315801/Put/seqid=0 2024-11-27T11:35:16,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741898_1074 (size=12301) 2024-11-27T11:35:16,794 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/201ac72000a346fb9a6b075976f85728 2024-11-27T11:35:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bdc7522936354df8828230dec3d36b87 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bdc7522936354df8828230dec3d36b87 2024-11-27T11:35:16,808 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bdc7522936354df8828230dec3d36b87, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T11:35:16,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d72c8fd2ba8549a784cfecc0cdcf35d9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d72c8fd2ba8549a784cfecc0cdcf35d9 2024-11-27T11:35:16,818 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d72c8fd2ba8549a784cfecc0cdcf35d9, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T11:35:16,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/201ac72000a346fb9a6b075976f85728 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/201ac72000a346fb9a6b075976f85728 2024-11-27T11:35:16,830 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/201ac72000a346fb9a6b075976f85728, entries=150, sequenceid=289, filesize=12.0 K 2024-11-27T11:35:16,832 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 080ce2238710db49165afb3a1a396a6b in 99ms, sequenceid=289, compaction requested=true 2024-11-27T11:35:16,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:16,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:16,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-27T11:35:16,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-27T11:35:16,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-27T11:35:16,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 717 msec 2024-11-27T11:35:16,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 726 msec 2024-11-27T11:35:16,998 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:16,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:16,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:35:17,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/84e76e9a3cf14c0393eb6aabc802e3da is 50, key is test_row_0/A:col10/1732707316998/Put/seqid=0 2024-11-27T11:35:17,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741899_1075 (size=14741) 2024-11-27T11:35:17,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/84e76e9a3cf14c0393eb6aabc802e3da 2024-11-27T11:35:17,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/e0dfeb9ae43040a49ac486482d1f814b is 50, key is test_row_0/B:col10/1732707316998/Put/seqid=0 2024-11-27T11:35:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741900_1076 (size=12301) 2024-11-27T11:35:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707377036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707377036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707377037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707377042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707377044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,130 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/9ee8f5dcac144db283bbd9b6766c05ea as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9ee8f5dcac144db283bbd9b6766c05ea 2024-11-27T11:35:17,134 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/37aa0b6b6c0841ad8ca8bee09deae2eb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/37aa0b6b6c0841ad8ca8bee09deae2eb 2024-11-27T11:35:17,137 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 9ee8f5dcac144db283bbd9b6766c05ea(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:17,137 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,137 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707316687; duration=0sec 2024-11-27T11:35:17,138 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:17,138 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:17,138 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:17,139 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:17,139 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:17,139 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,139 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/004997f048964f8e899c92bbb7e20c60, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/b719c2cbd1ac4c1eb5474a80a756bee2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/908d041f31da4dfa89db5bd0447df1cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/201ac72000a346fb9a6b075976f85728] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=48.3 K 2024-11-27T11:35:17,140 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004997f048964f8e899c92bbb7e20c60, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732707312265 2024-11-27T11:35:17,140 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b719c2cbd1ac4c1eb5474a80a756bee2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1732707313437 2024-11-27T11:35:17,141 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908d041f31da4dfa89db5bd0447df1cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732707314649 2024-11-27T11:35:17,141 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 201ac72000a346fb9a6b075976f85728, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732707315801 2024-11-27T11:35:17,146 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 37aa0b6b6c0841ad8ca8bee09deae2eb(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:17,146 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,146 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707316687; duration=0sec 2024-11-27T11:35:17,147 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:17,147 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:17,154 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#64 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,155 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/bbcf8050b9b44646a530a1d0395b72fd is 50, key is test_row_0/C:col10/1732707315801/Put/seqid=0 2024-11-27T11:35:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707377153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707377153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707377153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707377153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707377154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741901_1077 (size=12983) 2024-11-27T11:35:17,182 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/bbcf8050b9b44646a530a1d0395b72fd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/bbcf8050b9b44646a530a1d0395b72fd 2024-11-27T11:35:17,195 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into bbcf8050b9b44646a530a1d0395b72fd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:17,195 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,195 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=12, startTime=1732707316687; duration=0sec 2024-11-27T11:35:17,195 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:17,196 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:17,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-27T11:35:17,253 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:17,253 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:17,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-27T11:35:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T11:35:17,258 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:17,259 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:17,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:17,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T11:35:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707377363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707377363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707377363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707377363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707377363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T11:35:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:17,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:17,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:17,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:17,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/e0dfeb9ae43040a49ac486482d1f814b 2024-11-27T11:35:17,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f10b92b1c4944352b528db2fdf145374 is 50, key is test_row_0/C:col10/1732707316998/Put/seqid=0 2024-11-27T11:35:17,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741902_1078 (size=12301) 2024-11-27T11:35:17,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f10b92b1c4944352b528db2fdf145374 2024-11-27T11:35:17,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/84e76e9a3cf14c0393eb6aabc802e3da as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/84e76e9a3cf14c0393eb6aabc802e3da 2024-11-27T11:35:17,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/84e76e9a3cf14c0393eb6aabc802e3da, entries=200, sequenceid=302, filesize=14.4 K 2024-11-27T11:35:17,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/e0dfeb9ae43040a49ac486482d1f814b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/e0dfeb9ae43040a49ac486482d1f814b 2024-11-27T11:35:17,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/e0dfeb9ae43040a49ac486482d1f814b, entries=150, sequenceid=302, filesize=12.0 K 2024-11-27T11:35:17,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f10b92b1c4944352b528db2fdf145374 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f10b92b1c4944352b528db2fdf145374 2024-11-27T11:35:17,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f10b92b1c4944352b528db2fdf145374, entries=150, sequenceid=302, filesize=12.0 K 2024-11-27T11:35:17,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 080ce2238710db49165afb3a1a396a6b in 488ms, sequenceid=302, compaction requested=true 2024-11-27T11:35:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:17,486 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:17,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:17,487 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:17,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:17,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:17,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:17,489 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:17,489 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:17,489 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:17,489 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:17,489 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,489 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,489 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/37aa0b6b6c0841ad8ca8bee09deae2eb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d72c8fd2ba8549a784cfecc0cdcf35d9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/e0dfeb9ae43040a49ac486482d1f814b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=36.7 K 2024-11-27T11:35:17,489 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9ee8f5dcac144db283bbd9b6766c05ea, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bdc7522936354df8828230dec3d36b87, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/84e76e9a3cf14c0393eb6aabc802e3da] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=39.1 K 2024-11-27T11:35:17,489 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 37aa0b6b6c0841ad8ca8bee09deae2eb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732707314649 2024-11-27T11:35:17,489 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ee8f5dcac144db283bbd9b6766c05ea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1732707314649 2024-11-27T11:35:17,490 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d72c8fd2ba8549a784cfecc0cdcf35d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732707315801 2024-11-27T11:35:17,490 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdc7522936354df8828230dec3d36b87, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732707315801 2024-11-27T11:35:17,490 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e0dfeb9ae43040a49ac486482d1f814b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732707316966 2024-11-27T11:35:17,490 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e76e9a3cf14c0393eb6aabc802e3da, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732707316963 2024-11-27T11:35:17,498 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:17,498 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/201c25782f494a57ae27cc6a8f588887 is 50, key is test_row_0/B:col10/1732707316998/Put/seqid=0 2024-11-27T11:35:17,501 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#67 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:17,502 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8a359362756e4e29b069c31c1aae9650 is 50, key is test_row_0/A:col10/1732707316998/Put/seqid=0 2024-11-27T11:35:17,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741903_1079 (size=13051) 2024-11-27T11:35:17,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741904_1080 (size=13051) 2024-11-27T11:35:17,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-27T11:35:17,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,566 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:35:17,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T11:35:17,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/aba7e487dbbe4326b181f4bcfba907a3 is 50, key is test_row_0/A:col10/1732707317042/Put/seqid=0 2024-11-27T11:35:17,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741905_1081 (size=12301) 2024-11-27T11:35:17,578 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/aba7e487dbbe4326b181f4bcfba907a3 2024-11-27T11:35:17,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/90c4a04ebf9d451c85d4e2fd8cb47533 is 50, key is test_row_0/B:col10/1732707317042/Put/seqid=0 2024-11-27T11:35:17,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741906_1082 (size=12301) 2024-11-27T11:35:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/90c4a04ebf9d451c85d4e2fd8cb47533 2024-11-27T11:35:17,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/05c75617a20748a78806a1708b964392 is 50, key is test_row_0/C:col10/1732707317042/Put/seqid=0 2024-11-27T11:35:17,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741907_1083 (size=12301) 2024-11-27T11:35:17,621 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/05c75617a20748a78806a1708b964392 2024-11-27T11:35:17,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/aba7e487dbbe4326b181f4bcfba907a3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/aba7e487dbbe4326b181f4bcfba907a3 2024-11-27T11:35:17,636 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/aba7e487dbbe4326b181f4bcfba907a3, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T11:35:17,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/90c4a04ebf9d451c85d4e2fd8cb47533 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/90c4a04ebf9d451c85d4e2fd8cb47533 2024-11-27T11:35:17,644 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/90c4a04ebf9d451c85d4e2fd8cb47533, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T11:35:17,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/05c75617a20748a78806a1708b964392 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/05c75617a20748a78806a1708b964392 2024-11-27T11:35:17,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/05c75617a20748a78806a1708b964392, entries=150, sequenceid=328, filesize=12.0 K 2024-11-27T11:35:17,652 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 080ce2238710db49165afb3a1a396a6b in 86ms, sequenceid=328, compaction requested=true 2024-11-27T11:35:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-27T11:35:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-27T11:35:17,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-27T11:35:17,657 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 395 msec 2024-11-27T11:35:17,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 404 msec 2024-11-27T11:35:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:17,687 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:17,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:17,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3f4c551b6eae4324983e5b22e118c4c9 is 50, key is test_row_0/A:col10/1732707317685/Put/seqid=0 2024-11-27T11:35:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741908_1084 (size=22065) 2024-11-27T11:35:17,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3f4c551b6eae4324983e5b22e118c4c9 2024-11-27T11:35:17,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/f1eaa58ebbe944c6ab88a586cd6c2358 is 50, key is test_row_0/B:col10/1732707317685/Put/seqid=0 2024-11-27T11:35:17,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741909_1085 (size=12301) 2024-11-27T11:35:17,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/f1eaa58ebbe944c6ab88a586cd6c2358 2024-11-27T11:35:17,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707377724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707377724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,736 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707377727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/af2fd77c5c0c40568678099735dcdb2a is 50, key is test_row_0/C:col10/1732707317685/Put/seqid=0 2024-11-27T11:35:17,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707377728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707377730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,740 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741910_1086 (size=12301) 2024-11-27T11:35:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707377842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707377843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707377843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707377844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:17,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707377844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:17,845 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-27T11:35:17,883 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:17,883 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:17,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-27T11:35:17,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T11:35:17,886 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:17,887 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:17,888 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:17,923 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/201c25782f494a57ae27cc6a8f588887 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/201c25782f494a57ae27cc6a8f588887 2024-11-27T11:35:17,924 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8a359362756e4e29b069c31c1aae9650 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8a359362756e4e29b069c31c1aae9650 2024-11-27T11:35:17,932 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 8a359362756e4e29b069c31c1aae9650(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:17,932 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,933 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707317486; duration=0sec 2024-11-27T11:35:17,933 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:17,933 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:17,933 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:17,934 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:17,934 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:17,934 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:17,934 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/bbcf8050b9b44646a530a1d0395b72fd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f10b92b1c4944352b528db2fdf145374, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/05c75617a20748a78806a1708b964392] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=36.7 K 2024-11-27T11:35:17,934 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 201c25782f494a57ae27cc6a8f588887(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:17,935 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,935 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707317486; duration=0sec 2024-11-27T11:35:17,935 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbcf8050b9b44646a530a1d0395b72fd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732707315801 2024-11-27T11:35:17,935 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:17,935 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:17,935 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f10b92b1c4944352b528db2fdf145374, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732707316966 2024-11-27T11:35:17,935 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c75617a20748a78806a1708b964392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732707317035 2024-11-27T11:35:17,945 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:17,947 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/5ed9ccc0b4d24a30912bf0a2522a1e26 is 50, key is test_row_0/C:col10/1732707317042/Put/seqid=0 2024-11-27T11:35:17,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741911_1087 (size=13085) 2024-11-27T11:35:17,972 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/5ed9ccc0b4d24a30912bf0a2522a1e26 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5ed9ccc0b4d24a30912bf0a2522a1e26 2024-11-27T11:35:17,978 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 5ed9ccc0b4d24a30912bf0a2522a1e26(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:17,978 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:17,978 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707317487; duration=0sec 2024-11-27T11:35:17,978 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:17,978 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:17,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T11:35:18,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T11:35:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:18,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:18,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:18,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707378053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707378053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707378053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707378053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707378053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/af2fd77c5c0c40568678099735dcdb2a 2024-11-27T11:35:18,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3f4c551b6eae4324983e5b22e118c4c9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3f4c551b6eae4324983e5b22e118c4c9 2024-11-27T11:35:18,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3f4c551b6eae4324983e5b22e118c4c9, entries=350, sequenceid=340, filesize=21.5 K 2024-11-27T11:35:18,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/f1eaa58ebbe944c6ab88a586cd6c2358 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f1eaa58ebbe944c6ab88a586cd6c2358 2024-11-27T11:35:18,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f1eaa58ebbe944c6ab88a586cd6c2358, entries=150, sequenceid=340, filesize=12.0 K 2024-11-27T11:35:18,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/af2fd77c5c0c40568678099735dcdb2a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/af2fd77c5c0c40568678099735dcdb2a 2024-11-27T11:35:18,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/af2fd77c5c0c40568678099735dcdb2a, entries=150, sequenceid=340, filesize=12.0 K 2024-11-27T11:35:18,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 080ce2238710db49165afb3a1a396a6b in 490ms, sequenceid=340, compaction requested=true 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:18,178 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:18,178 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:18,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:18,179 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:18,179 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:18,179 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:18,179 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:18,179 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:18,180 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:18,180 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8a359362756e4e29b069c31c1aae9650, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/aba7e487dbbe4326b181f4bcfba907a3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3f4c551b6eae4324983e5b22e118c4c9] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=46.3 K 2024-11-27T11:35:18,180 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/201c25782f494a57ae27cc6a8f588887, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/90c4a04ebf9d451c85d4e2fd8cb47533, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f1eaa58ebbe944c6ab88a586cd6c2358] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=36.8 K 2024-11-27T11:35:18,180 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 201c25782f494a57ae27cc6a8f588887, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732707316966 2024-11-27T11:35:18,180 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a359362756e4e29b069c31c1aae9650, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732707316966 2024-11-27T11:35:18,181 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 90c4a04ebf9d451c85d4e2fd8cb47533, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732707317035 2024-11-27T11:35:18,181 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba7e487dbbe4326b181f4bcfba907a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732707317035 2024-11-27T11:35:18,182 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f1eaa58ebbe944c6ab88a586cd6c2358, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707317683 2024-11-27T11:35:18,182 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f4c551b6eae4324983e5b22e118c4c9, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707317674 2024-11-27T11:35:18,193 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:18,194 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#76 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:18,194 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8dc94f72d98b40c2813ceb50578832e6 is 50, key is test_row_0/A:col10/1732707317685/Put/seqid=0 2024-11-27T11:35:18,194 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5d1e4d63b45d416baded121973190054 is 50, key is test_row_0/B:col10/1732707317685/Put/seqid=0 2024-11-27T11:35:18,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-27T11:35:18,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:18,197 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:35:18,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T11:35:18,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6dad4856811e43ebaee482339a2f227b is 50, key is test_row_0/A:col10/1732707317724/Put/seqid=0 2024-11-27T11:35:18,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741913_1089 (size=13153) 2024-11-27T11:35:18,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741912_1088 (size=13153) 2024-11-27T11:35:18,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741914_1090 (size=12301) 2024-11-27T11:35:18,225 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6dad4856811e43ebaee482339a2f227b 2024-11-27T11:35:18,234 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/8dc94f72d98b40c2813ceb50578832e6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8dc94f72d98b40c2813ceb50578832e6 2024-11-27T11:35:18,243 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 8dc94f72d98b40c2813ceb50578832e6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:18,243 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:18,243 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707318178; duration=0sec 2024-11-27T11:35:18,243 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:18,243 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:18,243 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T11:35:18,245 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T11:35:18,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/0228e77f831e4b369e7ebbb2c4757a32 is 50, key is test_row_0/B:col10/1732707317724/Put/seqid=0 2024-11-27T11:35:18,245 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T11:35:18,245 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. because compaction request was cancelled 2024-11-27T11:35:18,245 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:18,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741915_1091 (size=12301) 2024-11-27T11:35:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:18,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:18,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707378383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707378385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707378385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707378386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707378386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707378492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707378493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707378493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707378503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707378503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T11:35:18,616 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5d1e4d63b45d416baded121973190054 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5d1e4d63b45d416baded121973190054 2024-11-27T11:35:18,622 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 5d1e4d63b45d416baded121973190054(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:18,622 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:18,622 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707318178; duration=0sec 2024-11-27T11:35:18,624 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:18,624 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:18,651 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/0228e77f831e4b369e7ebbb2c4757a32 2024-11-27T11:35:18,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/5c7b827f34ed49c1be6158a76010e00f is 50, key is test_row_0/C:col10/1732707317724/Put/seqid=0 2024-11-27T11:35:18,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741916_1092 (size=12301) 2024-11-27T11:35:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707378703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707378703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707378703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707378713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,715 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:18,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:18,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707378723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:18,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707379013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707379013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707379013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,017 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T11:35:19,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707379022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707379033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,079 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/5c7b827f34ed49c1be6158a76010e00f 2024-11-27T11:35:19,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6dad4856811e43ebaee482339a2f227b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6dad4856811e43ebaee482339a2f227b 2024-11-27T11:35:19,092 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6dad4856811e43ebaee482339a2f227b, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T11:35:19,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/0228e77f831e4b369e7ebbb2c4757a32 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0228e77f831e4b369e7ebbb2c4757a32 2024-11-27T11:35:19,098 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0228e77f831e4b369e7ebbb2c4757a32, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T11:35:19,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/5c7b827f34ed49c1be6158a76010e00f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5c7b827f34ed49c1be6158a76010e00f 2024-11-27T11:35:19,104 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5c7b827f34ed49c1be6158a76010e00f, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T11:35:19,106 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 080ce2238710db49165afb3a1a396a6b in 909ms, sequenceid=367, compaction requested=true 2024-11-27T11:35:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:19,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-27T11:35:19,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-27T11:35:19,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-27T11:35:19,111 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2200 sec 2024-11-27T11:35:19,114 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2300 sec 2024-11-27T11:35:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:19,526 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:19,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T11:35:19,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/848c05eaf4ad487e925a4bc73c8322fd is 50, key is test_row_0/A:col10/1732707318384/Put/seqid=0 2024-11-27T11:35:19,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741917_1093 (size=17181) 2024-11-27T11:35:19,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707379593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707379594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707379596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707379597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707379598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707379702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707379702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,703 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707379703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,704 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707379703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707379712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707379913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707379913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,919 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707379913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707379913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:19,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707379923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:19,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:19,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/848c05eaf4ad487e925a4bc73c8322fd 2024-11-27T11:35:19,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/00a8606678dc4ef3a78f07acd07c88ee is 50, key is test_row_0/B:col10/1732707318384/Put/seqid=0 2024-11-27T11:35:19,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741918_1094 (size=12301) 2024-11-27T11:35:20,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-27T11:35:20,033 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:20,033 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:20,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-27T11:35:20,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T11:35:20,036 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:20,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:20,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:20,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T11:35:20,190 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T11:35:20,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:20,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707380223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707380223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707380233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,234 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707380234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707380234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T11:35:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T11:35:20,362 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/00a8606678dc4ef3a78f07acd07c88ee 2024-11-27T11:35:20,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/95586e36015944509695ebe3ee068fc3 is 50, key is test_row_0/C:col10/1732707318384/Put/seqid=0 2024-11-27T11:35:20,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741919_1095 (size=12301) 2024-11-27T11:35:20,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T11:35:20,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T11:35:20,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T11:35:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707380732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,737 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707380732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,744 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707380743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707380743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707380752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:20,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/95586e36015944509695ebe3ee068fc3 2024-11-27T11:35:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/848c05eaf4ad487e925a4bc73c8322fd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/848c05eaf4ad487e925a4bc73c8322fd 2024-11-27T11:35:20,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/848c05eaf4ad487e925a4bc73c8322fd, entries=250, sequenceid=385, filesize=16.8 K 2024-11-27T11:35:20,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/00a8606678dc4ef3a78f07acd07c88ee as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/00a8606678dc4ef3a78f07acd07c88ee 2024-11-27T11:35:20,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T11:35:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:20,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:20,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/00a8606678dc4ef3a78f07acd07c88ee, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T11:35:20,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/95586e36015944509695ebe3ee068fc3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/95586e36015944509695ebe3ee068fc3 2024-11-27T11:35:20,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/95586e36015944509695ebe3ee068fc3, entries=150, sequenceid=385, filesize=12.0 K 2024-11-27T11:35:20,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 080ce2238710db49165afb3a1a396a6b in 1289ms, sequenceid=385, compaction requested=true 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:20,815 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:20,815 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:20,817 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:20,817 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:20,817 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:20,817 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:20,817 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,817 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,817 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5d1e4d63b45d416baded121973190054, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0228e77f831e4b369e7ebbb2c4757a32, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/00a8606678dc4ef3a78f07acd07c88ee] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=36.9 K 2024-11-27T11:35:20,817 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8dc94f72d98b40c2813ceb50578832e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6dad4856811e43ebaee482339a2f227b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/848c05eaf4ad487e925a4bc73c8322fd] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=41.6 K 2024-11-27T11:35:20,817 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d1e4d63b45d416baded121973190054, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707317683 2024-11-27T11:35:20,817 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dc94f72d98b40c2813ceb50578832e6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707317683 2024-11-27T11:35:20,818 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0228e77f831e4b369e7ebbb2c4757a32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732707317720 2024-11-27T11:35:20,818 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dad4856811e43ebaee482339a2f227b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732707317720 2024-11-27T11:35:20,818 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a8606678dc4ef3a78f07acd07c88ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732707318384 2024-11-27T11:35:20,818 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 848c05eaf4ad487e925a4bc73c8322fd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732707318383 2024-11-27T11:35:20,830 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#83 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:20,830 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/44ed600ab61a480baec87fc818f3350c is 50, key is test_row_0/B:col10/1732707318384/Put/seqid=0 2024-11-27T11:35:20,832 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:20,833 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/0a48b3c4aa1f49b9981ce2ad8f3875c0 is 50, key is test_row_0/A:col10/1732707318384/Put/seqid=0 2024-11-27T11:35:20,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741920_1096 (size=13255) 2024-11-27T11:35:20,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741921_1097 (size=13255) 2024-11-27T11:35:20,872 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/0a48b3c4aa1f49b9981ce2ad8f3875c0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0a48b3c4aa1f49b9981ce2ad8f3875c0 2024-11-27T11:35:20,881 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 0a48b3c4aa1f49b9981ce2ad8f3875c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:20,881 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:20,881 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707320815; duration=0sec 2024-11-27T11:35:20,881 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:20,882 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:20,882 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:20,883 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49988 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:20,883 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:20,884 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,884 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5ed9ccc0b4d24a30912bf0a2522a1e26, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/af2fd77c5c0c40568678099735dcdb2a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5c7b827f34ed49c1be6158a76010e00f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/95586e36015944509695ebe3ee068fc3] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=48.8 K 2024-11-27T11:35:20,885 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ed9ccc0b4d24a30912bf0a2522a1e26, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1732707317035 2024-11-27T11:35:20,886 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting af2fd77c5c0c40568678099735dcdb2a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707317683 2024-11-27T11:35:20,886 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c7b827f34ed49c1be6158a76010e00f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732707317720 2024-11-27T11:35:20,887 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95586e36015944509695ebe3ee068fc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732707318384 2024-11-27T11:35:20,898 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:20,899 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f780a4093f774b0389d784df1528c4df is 50, key is test_row_0/C:col10/1732707318384/Put/seqid=0 2024-11-27T11:35:20,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741922_1098 (size=13221) 2024-11-27T11:35:20,922 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f780a4093f774b0389d784df1528c4df as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f780a4093f774b0389d784df1528c4df 2024-11-27T11:35:20,933 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into f780a4093f774b0389d784df1528c4df(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:20,933 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:20,933 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=12, startTime=1732707320815; duration=0sec 2024-11-27T11:35:20,933 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:20,933 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:20,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-27T11:35:20,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:20,959 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T11:35:20,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6a75eb1c152744cf9f3b5a4532cc197c is 50, key is test_row_0/A:col10/1732707319594/Put/seqid=0 2024-11-27T11:35:21,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741923_1099 (size=12301) 2024-11-27T11:35:21,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T11:35:21,256 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/44ed600ab61a480baec87fc818f3350c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/44ed600ab61a480baec87fc818f3350c 2024-11-27T11:35:21,265 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 44ed600ab61a480baec87fc818f3350c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:21,265 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:21,265 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707320815; duration=0sec 2024-11-27T11:35:21,265 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:21,265 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:21,408 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6a75eb1c152744cf9f3b5a4532cc197c 2024-11-27T11:35:21,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/f7e1152db462429a992f3a98b5318f75 is 50, key is test_row_0/B:col10/1732707319594/Put/seqid=0 2024-11-27T11:35:21,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741924_1100 (size=12301) 2024-11-27T11:35:21,428 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/f7e1152db462429a992f3a98b5318f75 2024-11-27T11:35:21,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/6f5ff36af6d64f248f2d63196d7a594b is 50, key is test_row_0/C:col10/1732707319594/Put/seqid=0 2024-11-27T11:35:21,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741925_1101 (size=12301) 2024-11-27T11:35:21,444 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=407 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/6f5ff36af6d64f248f2d63196d7a594b 2024-11-27T11:35:21,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/6a75eb1c152744cf9f3b5a4532cc197c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6a75eb1c152744cf9f3b5a4532cc197c 2024-11-27T11:35:21,457 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6a75eb1c152744cf9f3b5a4532cc197c, entries=150, sequenceid=407, filesize=12.0 K 2024-11-27T11:35:21,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/f7e1152db462429a992f3a98b5318f75 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f7e1152db462429a992f3a98b5318f75 2024-11-27T11:35:21,466 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f7e1152db462429a992f3a98b5318f75, entries=150, sequenceid=407, filesize=12.0 K 2024-11-27T11:35:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/6f5ff36af6d64f248f2d63196d7a594b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6f5ff36af6d64f248f2d63196d7a594b 2024-11-27T11:35:21,474 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6f5ff36af6d64f248f2d63196d7a594b, entries=150, sequenceid=407, filesize=12.0 K 2024-11-27T11:35:21,475 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 080ce2238710db49165afb3a1a396a6b in 516ms, sequenceid=407, compaction requested=false 2024-11-27T11:35:21,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:21,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:21,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-27T11:35:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-27T11:35:21,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-27T11:35:21,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4400 sec 2024-11-27T11:35:21,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4550 sec 2024-11-27T11:35:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:21,772 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:21,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:21,786 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3b8f8ec6c4c54ed59f41fa287a81beff is 50, key is test_row_0/A:col10/1732707321769/Put/seqid=0 2024-11-27T11:35:21,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741926_1102 (size=12301) 2024-11-27T11:35:21,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707381810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707381811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707381812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707381813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707381814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707381922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707381922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707381922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,926 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707381933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707381933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:21,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707382132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707382132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707382133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707382143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707382143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-27T11:35:22,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:22,183 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:22,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-27T11:35:22,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T11:35:22,186 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:22,187 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:22,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:22,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3b8f8ec6c4c54ed59f41fa287a81beff 2024-11-27T11:35:22,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/4d6ec47e21604ef09a8a11865dbfe229 is 50, key is test_row_0/B:col10/1732707321769/Put/seqid=0 2024-11-27T11:35:22,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741927_1103 (size=12301) 2024-11-27T11:35:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T11:35:22,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T11:35:22,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707382443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707382443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707382443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707382452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,454 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707382452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T11:35:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,493 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T11:35:22,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/4d6ec47e21604ef09a8a11865dbfe229 2024-11-27T11:35:22,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/e91fd9b9df074b5f85371f47517ecc03 is 50, key is test_row_0/C:col10/1732707321769/Put/seqid=0 2024-11-27T11:35:22,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741928_1104 (size=12301) 2024-11-27T11:35:22,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T11:35:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T11:35:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:22,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T11:35:22,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T11:35:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:22,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:22,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707382952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707382952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707382952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707382963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707382963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:22,965 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:23,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/e91fd9b9df074b5f85371f47517ecc03 2024-11-27T11:35:23,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/3b8f8ec6c4c54ed59f41fa287a81beff as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3b8f8ec6c4c54ed59f41fa287a81beff 2024-11-27T11:35:23,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3b8f8ec6c4c54ed59f41fa287a81beff, entries=150, sequenceid=419, filesize=12.0 K 2024-11-27T11:35:23,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/4d6ec47e21604ef09a8a11865dbfe229 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/4d6ec47e21604ef09a8a11865dbfe229 2024-11-27T11:35:23,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/4d6ec47e21604ef09a8a11865dbfe229, entries=150, sequenceid=419, filesize=12.0 K 2024-11-27T11:35:23,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/e91fd9b9df074b5f85371f47517ecc03 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e91fd9b9df074b5f85371f47517ecc03 2024-11-27T11:35:23,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e91fd9b9df074b5f85371f47517ecc03, entries=150, sequenceid=419, filesize=12.0 K 2024-11-27T11:35:23,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 080ce2238710db49165afb3a1a396a6b in 1281ms, sequenceid=419, compaction requested=true 2024-11-27T11:35:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:23,053 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:23,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:23,053 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:23,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:23,055 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:23,055 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:23,055 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:23,055 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:23,055 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:23,055 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:23,055 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/44ed600ab61a480baec87fc818f3350c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f7e1152db462429a992f3a98b5318f75, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/4d6ec47e21604ef09a8a11865dbfe229] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.0 K 2024-11-27T11:35:23,055 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0a48b3c4aa1f49b9981ce2ad8f3875c0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6a75eb1c152744cf9f3b5a4532cc197c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3b8f8ec6c4c54ed59f41fa287a81beff] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.0 K 2024-11-27T11:35:23,055 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 44ed600ab61a480baec87fc818f3350c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732707318384 2024-11-27T11:35:23,055 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a48b3c4aa1f49b9981ce2ad8f3875c0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732707318384 2024-11-27T11:35:23,056 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f7e1152db462429a992f3a98b5318f75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732707319584 2024-11-27T11:35:23,056 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a75eb1c152744cf9f3b5a4532cc197c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732707319584 2024-11-27T11:35:23,056 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d6ec47e21604ef09a8a11865dbfe229, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732707321763 2024-11-27T11:35:23,056 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b8f8ec6c4c54ed59f41fa287a81beff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732707321763 2024-11-27T11:35:23,070 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:23,070 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:23,071 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/cb86a1ce49c8473b900af69dc733ffc7 is 50, key is test_row_0/B:col10/1732707321769/Put/seqid=0 2024-11-27T11:35:23,071 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/9484348d239d4cb989360868097b6463 is 50, key is test_row_0/A:col10/1732707321769/Put/seqid=0 2024-11-27T11:35:23,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741929_1105 (size=13357) 2024-11-27T11:35:23,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741930_1106 (size=13357) 2024-11-27T11:35:23,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-27T11:35:23,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:23,108 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:35:23,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/eb3d26c992c847bb93d02223fe643a6f is 50, key is test_row_0/A:col10/1732707321807/Put/seqid=0 2024-11-27T11:35:23,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741931_1107 (size=12301) 2024-11-27T11:35:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T11:35:23,483 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/cb86a1ce49c8473b900af69dc733ffc7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cb86a1ce49c8473b900af69dc733ffc7 2024-11-27T11:35:23,485 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/9484348d239d4cb989360868097b6463 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9484348d239d4cb989360868097b6463 2024-11-27T11:35:23,490 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 9484348d239d4cb989360868097b6463(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:23,490 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into cb86a1ce49c8473b900af69dc733ffc7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:23,490 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707323053; duration=0sec 2024-11-27T11:35:23,490 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707323053; duration=0sec 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:23,490 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:23,491 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:23,492 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:23,492 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:23,492 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f780a4093f774b0389d784df1528c4df, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6f5ff36af6d64f248f2d63196d7a594b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e91fd9b9df074b5f85371f47517ecc03] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=36.9 K 2024-11-27T11:35:23,492 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f780a4093f774b0389d784df1528c4df, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1732707318384 2024-11-27T11:35:23,493 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f5ff36af6d64f248f2d63196d7a594b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=407, earliestPutTs=1732707319584 2024-11-27T11:35:23,493 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e91fd9b9df074b5f85371f47517ecc03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732707321763 2024-11-27T11:35:23,501 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:23,502 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/787445c555eb4d40a1360c76243e633c is 50, key is test_row_0/C:col10/1732707321769/Put/seqid=0 2024-11-27T11:35:23,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741932_1108 (size=13323) 2024-11-27T11:35:23,520 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/eb3d26c992c847bb93d02223fe643a6f 2024-11-27T11:35:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/fe3313cc37d54fec82942e45b1089575 is 50, key is test_row_0/B:col10/1732707321807/Put/seqid=0 2024-11-27T11:35:23,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741933_1109 (size=12301) 2024-11-27T11:35:23,918 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/787445c555eb4d40a1360c76243e633c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/787445c555eb4d40a1360c76243e633c 2024-11-27T11:35:23,925 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 787445c555eb4d40a1360c76243e633c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:23,925 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:23,925 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707323053; duration=0sec 2024-11-27T11:35:23,925 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:23,925 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:23,934 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/fe3313cc37d54fec82942e45b1089575 2024-11-27T11:35:23,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/9a5e5325b8f644288a3dc0b63c1424d4 is 50, key is test_row_0/C:col10/1732707321807/Put/seqid=0 2024-11-27T11:35:23,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741934_1110 (size=12301) 2024-11-27T11:35:23,957 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/9a5e5325b8f644288a3dc0b63c1424d4 2024-11-27T11:35:23,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/eb3d26c992c847bb93d02223fe643a6f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/eb3d26c992c847bb93d02223fe643a6f 2024-11-27T11:35:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:23,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:23,970 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/eb3d26c992c847bb93d02223fe643a6f, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T11:35:23,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/fe3313cc37d54fec82942e45b1089575 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fe3313cc37d54fec82942e45b1089575 2024-11-27T11:35:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:23,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35034 deadline: 1732707383973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:23,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707383977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:23,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:23,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707383978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:23,980 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in 080ce2238710db49165afb3a1a396a6b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,980 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:23,981 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fe3313cc37d54fec82942e45b1089575, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T11:35:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707383978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:23,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707383981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:23,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/9a5e5325b8f644288a3dc0b63c1424d4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/9a5e5325b8f644288a3dc0b63c1424d4 2024-11-27T11:35:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:23,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:23,986 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/9a5e5325b8f644288a3dc0b63c1424d4, entries=150, sequenceid=445, filesize=12.0 K 2024-11-27T11:35:23,987 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 080ce2238710db49165afb3a1a396a6b in 879ms, sequenceid=445, compaction requested=false 2024-11-27T11:35:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:23,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-27T11:35:23,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-27T11:35:23,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-27T11:35:23,991 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8020 sec 2024-11-27T11:35:23,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8080 sec 2024-11-27T11:35:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:24,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:24,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:24,101 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bf9038eff13542a286cd68edadf9d635 is 50, key is test_row_0/A:col10/1732707324094/Put/seqid=0 2024-11-27T11:35:24,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741935_1111 (size=12301) 2024-11-27T11:35:24,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707384131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707384133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707384140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707384142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707384252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707384253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707384253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707384253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-27T11:35:24,333 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:24,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-27T11:35:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T11:35:24,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:24,336 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:24,336 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:24,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T11:35:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707384463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707384463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707384463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707384463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,487 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:24,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:24,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,488 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bf9038eff13542a286cd68edadf9d635 2024-11-27T11:35:24,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d73748137f224485bc7003058eb8ed89 is 50, key is test_row_0/B:col10/1732707324094/Put/seqid=0 2024-11-27T11:35:24,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741936_1112 (size=12301) 2024-11-27T11:35:24,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:24,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T11:35:24,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707384773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707384773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707384782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707384782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:24,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:24,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:24,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:24,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:24,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:24,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d73748137f224485bc7003058eb8ed89 2024-11-27T11:35:24,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/7e55108592de46c4801f4a3675524629 is 50, key is test_row_0/C:col10/1732707324094/Put/seqid=0 2024-11-27T11:35:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T11:35:24,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741937_1113 (size=12301) 2024-11-27T11:35:25,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:25,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,102 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:25,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:25,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:25,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:25,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:25,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:25,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35094 deadline: 1732707385282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:25,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35076 deadline: 1732707385282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,285 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:25,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:25,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35068 deadline: 1732707385292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:25,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:25,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35044 deadline: 1732707385292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:35:25,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/7e55108592de46c4801f4a3675524629 2024-11-27T11:35:25,375 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/bf9038eff13542a286cd68edadf9d635 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bf9038eff13542a286cd68edadf9d635 2024-11-27T11:35:25,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bf9038eff13542a286cd68edadf9d635, entries=150, sequenceid=460, filesize=12.0 K 2024-11-27T11:35:25,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d73748137f224485bc7003058eb8ed89 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d73748137f224485bc7003058eb8ed89 2024-11-27T11:35:25,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d73748137f224485bc7003058eb8ed89, entries=150, sequenceid=460, filesize=12.0 K 2024-11-27T11:35:25,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/7e55108592de46c4801f4a3675524629 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7e55108592de46c4801f4a3675524629 2024-11-27T11:35:25,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7e55108592de46c4801f4a3675524629, entries=150, sequenceid=460, filesize=12.0 K 2024-11-27T11:35:25,392 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 080ce2238710db49165afb3a1a396a6b in 1298ms, sequenceid=460, compaction requested=true 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:25,392 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 080ce2238710db49165afb3a1a396a6b:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:25,392 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:25,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:25,393 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:25,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:25,394 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/A is initiating minor compaction (all files) 2024-11-27T11:35:25,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/B is initiating minor compaction (all files) 2024-11-27T11:35:25,394 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/A in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,394 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/B in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,394 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9484348d239d4cb989360868097b6463, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/eb3d26c992c847bb93d02223fe643a6f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bf9038eff13542a286cd68edadf9d635] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.1 K 2024-11-27T11:35:25,394 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cb86a1ce49c8473b900af69dc733ffc7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fe3313cc37d54fec82942e45b1089575, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d73748137f224485bc7003058eb8ed89] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.1 K 2024-11-27T11:35:25,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting cb86a1ce49c8473b900af69dc733ffc7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732707321763 2024-11-27T11:35:25,394 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9484348d239d4cb989360868097b6463, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732707321763 2024-11-27T11:35:25,396 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting fe3313cc37d54fec82942e45b1089575, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732707321807 2024-11-27T11:35:25,396 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb3d26c992c847bb93d02223fe643a6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732707321807 2024-11-27T11:35:25,397 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d73748137f224485bc7003058eb8ed89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732707323976 2024-11-27T11:35:25,397 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9038eff13542a286cd68edadf9d635, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732707323976 2024-11-27T11:35:25,405 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#B#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:25,405 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5ea3b84610a44b0caec751dab5dcadf9 is 50, key is test_row_0/B:col10/1732707324094/Put/seqid=0 2024-11-27T11:35:25,407 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#A#compaction#102 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:25,407 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/4cc39de433254550ab8bf464d6061230 is 50, key is test_row_0/A:col10/1732707324094/Put/seqid=0 2024-11-27T11:35:25,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-27T11:35:25,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,408 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:35:25,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741938_1114 (size=13459) 2024-11-27T11:35:25,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/d6e33e03058445a684ff064e98519daa is 50, key is test_row_0/A:col10/1732707324136/Put/seqid=0 2024-11-27T11:35:25,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741939_1115 (size=13459) 2024-11-27T11:35:25,432 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/4cc39de433254550ab8bf464d6061230 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4cc39de433254550ab8bf464d6061230 2024-11-27T11:35:25,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741940_1116 (size=12301) 2024-11-27T11:35:25,440 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/d6e33e03058445a684ff064e98519daa 2024-11-27T11:35:25,442 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/A of 080ce2238710db49165afb3a1a396a6b into 4cc39de433254550ab8bf464d6061230(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:25,442 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:25,442 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/A, priority=13, startTime=1732707325392; duration=0sec 2024-11-27T11:35:25,442 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:25,442 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:A 2024-11-27T11:35:25,442 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:25,445 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:25,445 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 080ce2238710db49165afb3a1a396a6b/C is initiating minor compaction (all files) 2024-11-27T11:35:25,446 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 080ce2238710db49165afb3a1a396a6b/C in TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:25,446 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/787445c555eb4d40a1360c76243e633c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/9a5e5325b8f644288a3dc0b63c1424d4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7e55108592de46c4801f4a3675524629] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp, totalSize=37.0 K 2024-11-27T11:35:25,446 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787445c555eb4d40a1360c76243e633c, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732707321763 2024-11-27T11:35:25,448 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a5e5325b8f644288a3dc0b63c1424d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1732707321807 2024-11-27T11:35:25,448 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e55108592de46c4801f4a3675524629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1732707323976 2024-11-27T11:35:25,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d0686fecc0254271b2513458fc572eff is 50, key is test_row_0/B:col10/1732707324136/Put/seqid=0 2024-11-27T11:35:25,459 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 080ce2238710db49165afb3a1a396a6b#C#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:25,460 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/78324f30c3604d73a8f1fd8443df14ca is 50, key is test_row_0/C:col10/1732707324094/Put/seqid=0 2024-11-27T11:35:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741941_1117 (size=12301) 2024-11-27T11:35:25,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741942_1118 (size=13425) 2024-11-27T11:35:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T11:35:25,486 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/78324f30c3604d73a8f1fd8443df14ca as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/78324f30c3604d73a8f1fd8443df14ca 2024-11-27T11:35:25,503 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/C of 080ce2238710db49165afb3a1a396a6b into 78324f30c3604d73a8f1fd8443df14ca(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:25,503 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:25,503 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/C, priority=13, startTime=1732707325392; duration=0sec 2024-11-27T11:35:25,503 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:25,503 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:C 2024-11-27T11:35:25,601 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-27T11:35:25,601 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:25,603 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-27T11:35:25,603 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-27T11:35:25,603 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:25,603 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:25,605 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-27T11:35:25,605 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:25,606 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,606 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,606 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,606 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,607 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,607 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:25,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:25,607 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:25,607 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,608 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,608 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:25,822 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/5ea3b84610a44b0caec751dab5dcadf9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5ea3b84610a44b0caec751dab5dcadf9 2024-11-27T11:35:25,828 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 080ce2238710db49165afb3a1a396a6b/B of 080ce2238710db49165afb3a1a396a6b into 5ea3b84610a44b0caec751dab5dcadf9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:25,828 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:25,828 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., storeName=080ce2238710db49165afb3a1a396a6b/B, priority=13, startTime=1732707325392; duration=0sec 2024-11-27T11:35:25,828 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:25,828 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 080ce2238710db49165afb3a1a396a6b:B 2024-11-27T11:35:25,869 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d0686fecc0254271b2513458fc572eff 2024-11-27T11:35:25,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f6670f143d9b45539b477993d87b7491 is 50, key is test_row_0/C:col10/1732707324136/Put/seqid=0 2024-11-27T11:35:25,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741943_1119 (size=12301) 2024-11-27T11:35:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:25,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. as already flushing 2024-11-27T11:35:25,994 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-27T11:35:25,994 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:25,994 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,994 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:25,994 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:26,282 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f6670f143d9b45539b477993d87b7491 2024-11-27T11:35:26,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/d6e33e03058445a684ff064e98519daa as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d6e33e03058445a684ff064e98519daa 2024-11-27T11:35:26,291 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d6e33e03058445a684ff064e98519daa, entries=150, sequenceid=484, filesize=12.0 K 2024-11-27T11:35:26,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/d0686fecc0254271b2513458fc572eff as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d0686fecc0254271b2513458fc572eff 2024-11-27T11:35:26,296 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d0686fecc0254271b2513458fc572eff, entries=150, sequenceid=484, filesize=12.0 K 2024-11-27T11:35:26,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/f6670f143d9b45539b477993d87b7491 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f6670f143d9b45539b477993d87b7491 2024-11-27T11:35:26,300 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f6670f143d9b45539b477993d87b7491, entries=150, sequenceid=484, filesize=12.0 K 2024-11-27T11:35:26,301 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for 080ce2238710db49165afb3a1a396a6b in 893ms, sequenceid=484, compaction requested=false 2024-11-27T11:35:26,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 080ce2238710db49165afb3a1a396a6b: 2024-11-27T11:35:26,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:26,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-27T11:35:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-27T11:35:26,304 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-27T11:35:26,304 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:26,304 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,304 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-27T11:35:26,304 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,304 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:26,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:26,304 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,304 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,304 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:26,305 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-27T11:35:26,305 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:26,305 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-27T11:35:26,305 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,305 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9670 sec 2024-11-27T11:35:26,305 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:26,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.9720 sec 2024-11-27T11:35:26,314 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-27T11:35:26,314 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:26,314 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,314 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,314 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-27T11:35:26,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 101 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3447 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3412 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1531 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4584 rows 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1532 2024-11-27T11:35:26,483 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4593 rows 2024-11-27T11:35:26,483 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:35:26,483 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:35:26,483 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:26,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:26,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T11:35:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T11:35:26,497 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707326496"}]},"ts":"1732707326496"} 2024-11-27T11:35:26,499 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T11:35:26,499 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T11:35:26,501 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:35:26,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, UNASSIGN}] 2024-11-27T11:35:26,507 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, UNASSIGN 2024-11-27T11:35:26,508 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=080ce2238710db49165afb3a1a396a6b, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:26,510 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, UNASSIGN because future has completed 2024-11-27T11:35:26,510 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:35:26,511 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:26,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T11:35:26,668 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:26,668 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:35:26,668 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 080ce2238710db49165afb3a1a396a6b, disabling compactions & flushes 2024-11-27T11:35:26,668 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:26,668 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:26,669 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. after waiting 0 ms 2024-11-27T11:35:26,669 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:26,669 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 080ce2238710db49165afb3a1a396a6b 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T11:35:26,673 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/4a0c87bbd8d1460586ff1822cefdce2a is 50, key is test_row_0/A:col10/1732707326303/Put/seqid=0 2024-11-27T11:35:26,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741944_1120 (size=12301) 2024-11-27T11:35:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T11:35:27,078 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/4a0c87bbd8d1460586ff1822cefdce2a 2024-11-27T11:35:27,085 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/2d8f104c1c474f25a7d3cb660ad90c20 is 50, key is test_row_0/B:col10/1732707326303/Put/seqid=0 2024-11-27T11:35:27,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741945_1121 (size=12301) 2024-11-27T11:35:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T11:35:27,432 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T11:35:27,489 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/2d8f104c1c474f25a7d3cb660ad90c20 2024-11-27T11:35:27,497 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/3222f0e794074155b98147a41dd02e30 is 50, key is test_row_0/C:col10/1732707326303/Put/seqid=0 2024-11-27T11:35:27,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741946_1122 (size=12301) 2024-11-27T11:35:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T11:35:27,901 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/3222f0e794074155b98147a41dd02e30 2024-11-27T11:35:27,906 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/A/4a0c87bbd8d1460586ff1822cefdce2a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4a0c87bbd8d1460586ff1822cefdce2a 2024-11-27T11:35:27,910 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4a0c87bbd8d1460586ff1822cefdce2a, entries=150, sequenceid=495, filesize=12.0 K 2024-11-27T11:35:27,911 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/B/2d8f104c1c474f25a7d3cb660ad90c20 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/2d8f104c1c474f25a7d3cb660ad90c20 2024-11-27T11:35:27,915 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/2d8f104c1c474f25a7d3cb660ad90c20, entries=150, sequenceid=495, filesize=12.0 K 2024-11-27T11:35:27,916 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/.tmp/C/3222f0e794074155b98147a41dd02e30 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/3222f0e794074155b98147a41dd02e30 2024-11-27T11:35:27,920 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/3222f0e794074155b98147a41dd02e30, entries=150, sequenceid=495, filesize=12.0 K 2024-11-27T11:35:27,921 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 080ce2238710db49165afb3a1a396a6b in 1252ms, sequenceid=495, compaction requested=true 2024-11-27T11:35:27,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0574e36323da479ba5934d72ad2e1409, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/cc35374792ea4a41b7821a3ed36efe21, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d88bfcfc59f0469783c2696e6f8987a7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/57e760a789bb4a4791a85e07275bba1f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/a7ef85be98204c5dad438c140c896a68, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/f268aacc86584ea7a72d621e0edaa54c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/13ec9a9bad0548fbb010dc979a6be414, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/c258c37cf5b049d4942a25434350f682, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bba5f8b0d54b41f688b344f8de9e3387, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/277631b0e315411db50f599e4688563a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6ece38beeeec46d7810c8983c1a154b0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/08859fb86c9245b3ad495d3eeefbf882, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6e54141c548746be81616edfd4e32c3f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8003e345c5c147f680b6304984c67064, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/77f1b9eb94024e76a16ed7331d0d6750, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/7e9ac9b5232e41bb887e93ed6f664c32, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/1c05fbad67f94d3ebb2ce8b54b0390a4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/42bf1c84d3854e789c36bcdb8a98cec2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3bac3cb8c57348dea9a8c1a3c2c6faf3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9ee8f5dcac144db283bbd9b6766c05ea, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bdc7522936354df8828230dec3d36b87, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/84e76e9a3cf14c0393eb6aabc802e3da, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8a359362756e4e29b069c31c1aae9650, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/aba7e487dbbe4326b181f4bcfba907a3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3f4c551b6eae4324983e5b22e118c4c9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8dc94f72d98b40c2813ceb50578832e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6dad4856811e43ebaee482339a2f227b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/848c05eaf4ad487e925a4bc73c8322fd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0a48b3c4aa1f49b9981ce2ad8f3875c0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6a75eb1c152744cf9f3b5a4532cc197c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9484348d239d4cb989360868097b6463, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3b8f8ec6c4c54ed59f41fa287a81beff, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/eb3d26c992c847bb93d02223fe643a6f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bf9038eff13542a286cd68edadf9d635] to archive 2024-11-27T11:35:27,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:35:27,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0574e36323da479ba5934d72ad2e1409 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0574e36323da479ba5934d72ad2e1409 2024-11-27T11:35:27,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/cc35374792ea4a41b7821a3ed36efe21 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/cc35374792ea4a41b7821a3ed36efe21 2024-11-27T11:35:27,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d88bfcfc59f0469783c2696e6f8987a7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d88bfcfc59f0469783c2696e6f8987a7 2024-11-27T11:35:27,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/57e760a789bb4a4791a85e07275bba1f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/57e760a789bb4a4791a85e07275bba1f 2024-11-27T11:35:27,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/a7ef85be98204c5dad438c140c896a68 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/a7ef85be98204c5dad438c140c896a68 2024-11-27T11:35:27,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/f268aacc86584ea7a72d621e0edaa54c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/f268aacc86584ea7a72d621e0edaa54c 2024-11-27T11:35:27,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/13ec9a9bad0548fbb010dc979a6be414 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/13ec9a9bad0548fbb010dc979a6be414 2024-11-27T11:35:27,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/c258c37cf5b049d4942a25434350f682 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/c258c37cf5b049d4942a25434350f682 2024-11-27T11:35:27,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bba5f8b0d54b41f688b344f8de9e3387 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bba5f8b0d54b41f688b344f8de9e3387 2024-11-27T11:35:27,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/277631b0e315411db50f599e4688563a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/277631b0e315411db50f599e4688563a 2024-11-27T11:35:27,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6ece38beeeec46d7810c8983c1a154b0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6ece38beeeec46d7810c8983c1a154b0 2024-11-27T11:35:27,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/08859fb86c9245b3ad495d3eeefbf882 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/08859fb86c9245b3ad495d3eeefbf882 2024-11-27T11:35:27,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6e54141c548746be81616edfd4e32c3f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6e54141c548746be81616edfd4e32c3f 2024-11-27T11:35:27,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8003e345c5c147f680b6304984c67064 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8003e345c5c147f680b6304984c67064 2024-11-27T11:35:27,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/77f1b9eb94024e76a16ed7331d0d6750 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/77f1b9eb94024e76a16ed7331d0d6750 2024-11-27T11:35:27,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/7e9ac9b5232e41bb887e93ed6f664c32 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/7e9ac9b5232e41bb887e93ed6f664c32 2024-11-27T11:35:27,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/1c05fbad67f94d3ebb2ce8b54b0390a4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/1c05fbad67f94d3ebb2ce8b54b0390a4 2024-11-27T11:35:27,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/42bf1c84d3854e789c36bcdb8a98cec2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/42bf1c84d3854e789c36bcdb8a98cec2 2024-11-27T11:35:27,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3bac3cb8c57348dea9a8c1a3c2c6faf3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3bac3cb8c57348dea9a8c1a3c2c6faf3 2024-11-27T11:35:27,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9ee8f5dcac144db283bbd9b6766c05ea to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9ee8f5dcac144db283bbd9b6766c05ea 2024-11-27T11:35:27,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bdc7522936354df8828230dec3d36b87 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bdc7522936354df8828230dec3d36b87 2024-11-27T11:35:27,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/84e76e9a3cf14c0393eb6aabc802e3da to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/84e76e9a3cf14c0393eb6aabc802e3da 2024-11-27T11:35:27,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8a359362756e4e29b069c31c1aae9650 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8a359362756e4e29b069c31c1aae9650 2024-11-27T11:35:27,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/aba7e487dbbe4326b181f4bcfba907a3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/aba7e487dbbe4326b181f4bcfba907a3 2024-11-27T11:35:27,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3f4c551b6eae4324983e5b22e118c4c9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3f4c551b6eae4324983e5b22e118c4c9 2024-11-27T11:35:27,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8dc94f72d98b40c2813ceb50578832e6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/8dc94f72d98b40c2813ceb50578832e6 2024-11-27T11:35:27,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6dad4856811e43ebaee482339a2f227b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6dad4856811e43ebaee482339a2f227b 2024-11-27T11:35:27,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/848c05eaf4ad487e925a4bc73c8322fd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/848c05eaf4ad487e925a4bc73c8322fd 2024-11-27T11:35:27,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0a48b3c4aa1f49b9981ce2ad8f3875c0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/0a48b3c4aa1f49b9981ce2ad8f3875c0 2024-11-27T11:35:27,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6a75eb1c152744cf9f3b5a4532cc197c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/6a75eb1c152744cf9f3b5a4532cc197c 2024-11-27T11:35:27,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9484348d239d4cb989360868097b6463 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/9484348d239d4cb989360868097b6463 2024-11-27T11:35:27,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3b8f8ec6c4c54ed59f41fa287a81beff to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/3b8f8ec6c4c54ed59f41fa287a81beff 2024-11-27T11:35:27,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/eb3d26c992c847bb93d02223fe643a6f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/eb3d26c992c847bb93d02223fe643a6f 2024-11-27T11:35:27,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bf9038eff13542a286cd68edadf9d635 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/bf9038eff13542a286cd68edadf9d635 2024-11-27T11:35:27,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/154896dc1fc844448e92de5effeac903, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a6b3a011509148aaa650a79252de79ea, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6d8b7c5bba7742f0817809c5d3b5c267, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5795e02db8ec44b9bc5771288cd6377e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6839c35c433849349dfd63dafad821b1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a8263c7edcad4d3fbed077d8c1bc94b9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/08c93eca60fc4d169d9971d402feca8d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/55c4f8ef3f9b46829227467381f0286f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cc8bcc9017b04964b911184508f9a2c9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/df9253a1d5c645c6a89829df0857b535, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ecd7e0f03b5d48a79ed479bc11c23b3d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/91bf66c703114bf798d6e85b2334fe91, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0bb381a8d4c7488192b3ada902accc6b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fed9ca3c35ba4e01bc322388b052d9e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5a6e85fc31f34456b941cb0f10eef429, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ea59a0f714c543a4877b7744b49106d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d3345ca27e424d94b81fa4cbb7eb155b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/360f1c912db0456b80c23ce6e564ade2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/37aa0b6b6c0841ad8ca8bee09deae2eb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a22078f15c4c4094b63dac5cbb025c01, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d72c8fd2ba8549a784cfecc0cdcf35d9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/201c25782f494a57ae27cc6a8f588887, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/e0dfeb9ae43040a49ac486482d1f814b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/90c4a04ebf9d451c85d4e2fd8cb47533, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5d1e4d63b45d416baded121973190054, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f1eaa58ebbe944c6ab88a586cd6c2358, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0228e77f831e4b369e7ebbb2c4757a32, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/44ed600ab61a480baec87fc818f3350c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/00a8606678dc4ef3a78f07acd07c88ee, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f7e1152db462429a992f3a98b5318f75, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cb86a1ce49c8473b900af69dc733ffc7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/4d6ec47e21604ef09a8a11865dbfe229, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fe3313cc37d54fec82942e45b1089575, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d73748137f224485bc7003058eb8ed89] to archive 2024-11-27T11:35:27,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:35:27,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/154896dc1fc844448e92de5effeac903 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/154896dc1fc844448e92de5effeac903 2024-11-27T11:35:27,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a6b3a011509148aaa650a79252de79ea to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a6b3a011509148aaa650a79252de79ea 2024-11-27T11:35:27,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6d8b7c5bba7742f0817809c5d3b5c267 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6d8b7c5bba7742f0817809c5d3b5c267 2024-11-27T11:35:27,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5795e02db8ec44b9bc5771288cd6377e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5795e02db8ec44b9bc5771288cd6377e 2024-11-27T11:35:27,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6839c35c433849349dfd63dafad821b1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/6839c35c433849349dfd63dafad821b1 2024-11-27T11:35:27,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a8263c7edcad4d3fbed077d8c1bc94b9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a8263c7edcad4d3fbed077d8c1bc94b9 2024-11-27T11:35:27,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/08c93eca60fc4d169d9971d402feca8d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/08c93eca60fc4d169d9971d402feca8d 2024-11-27T11:35:27,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/55c4f8ef3f9b46829227467381f0286f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/55c4f8ef3f9b46829227467381f0286f 2024-11-27T11:35:27,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cc8bcc9017b04964b911184508f9a2c9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cc8bcc9017b04964b911184508f9a2c9 2024-11-27T11:35:27,993 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/df9253a1d5c645c6a89829df0857b535 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/df9253a1d5c645c6a89829df0857b535 2024-11-27T11:35:27,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ecd7e0f03b5d48a79ed479bc11c23b3d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ecd7e0f03b5d48a79ed479bc11c23b3d 2024-11-27T11:35:27,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/91bf66c703114bf798d6e85b2334fe91 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/91bf66c703114bf798d6e85b2334fe91 2024-11-27T11:35:27,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0bb381a8d4c7488192b3ada902accc6b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0bb381a8d4c7488192b3ada902accc6b 2024-11-27T11:35:27,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fed9ca3c35ba4e01bc322388b052d9e6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fed9ca3c35ba4e01bc322388b052d9e6 2024-11-27T11:35:27,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5a6e85fc31f34456b941cb0f10eef429 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5a6e85fc31f34456b941cb0f10eef429 2024-11-27T11:35:27,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ea59a0f714c543a4877b7744b49106d6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/ea59a0f714c543a4877b7744b49106d6 2024-11-27T11:35:28,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d3345ca27e424d94b81fa4cbb7eb155b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d3345ca27e424d94b81fa4cbb7eb155b 2024-11-27T11:35:28,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/360f1c912db0456b80c23ce6e564ade2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/360f1c912db0456b80c23ce6e564ade2 2024-11-27T11:35:28,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/37aa0b6b6c0841ad8ca8bee09deae2eb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/37aa0b6b6c0841ad8ca8bee09deae2eb 2024-11-27T11:35:28,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a22078f15c4c4094b63dac5cbb025c01 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/a22078f15c4c4094b63dac5cbb025c01 2024-11-27T11:35:28,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d72c8fd2ba8549a784cfecc0cdcf35d9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d72c8fd2ba8549a784cfecc0cdcf35d9 2024-11-27T11:35:28,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/201c25782f494a57ae27cc6a8f588887 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/201c25782f494a57ae27cc6a8f588887 2024-11-27T11:35:28,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/e0dfeb9ae43040a49ac486482d1f814b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/e0dfeb9ae43040a49ac486482d1f814b 2024-11-27T11:35:28,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/90c4a04ebf9d451c85d4e2fd8cb47533 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/90c4a04ebf9d451c85d4e2fd8cb47533 2024-11-27T11:35:28,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5d1e4d63b45d416baded121973190054 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5d1e4d63b45d416baded121973190054 2024-11-27T11:35:28,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f1eaa58ebbe944c6ab88a586cd6c2358 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f1eaa58ebbe944c6ab88a586cd6c2358 2024-11-27T11:35:28,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0228e77f831e4b369e7ebbb2c4757a32 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/0228e77f831e4b369e7ebbb2c4757a32 2024-11-27T11:35:28,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/44ed600ab61a480baec87fc818f3350c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/44ed600ab61a480baec87fc818f3350c 2024-11-27T11:35:28,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/00a8606678dc4ef3a78f07acd07c88ee to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/00a8606678dc4ef3a78f07acd07c88ee 2024-11-27T11:35:28,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f7e1152db462429a992f3a98b5318f75 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/f7e1152db462429a992f3a98b5318f75 2024-11-27T11:35:28,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cb86a1ce49c8473b900af69dc733ffc7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/cb86a1ce49c8473b900af69dc733ffc7 2024-11-27T11:35:28,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/4d6ec47e21604ef09a8a11865dbfe229 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/4d6ec47e21604ef09a8a11865dbfe229 2024-11-27T11:35:28,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fe3313cc37d54fec82942e45b1089575 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/fe3313cc37d54fec82942e45b1089575 2024-11-27T11:35:28,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d73748137f224485bc7003058eb8ed89 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d73748137f224485bc7003058eb8ed89 2024-11-27T11:35:28,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6705caa12e3f41c387922e628d8293c8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2465cba93a3f479e93073e61934525d4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7455744b98eb4ab1a5660fa6406d540b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/eaff47fd114e4f77bfe3c7ee676632e3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/a4a56ef44bd14d52b1c6ca47e2358e7d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/270cd73c51f04ad989c5d6c7dd3e28c1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/fe78dd5902684afdac852d315800f38b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/17da9b666b84417c90263eba2c4d179a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/019ea283a1e34bc48879dc6d87467288, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/1acaa3fe7ae445328be87f86a32377fb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f1b9f029be364fc9abf2d204fea4db0a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e8f8c1e6b4334777911fffe06dbafa02, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2da3c5ed04cd46b686e040ce46e33b97, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2865e16edfc94b84a893a45b966458d8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/c851fca7a6b44baaa1a00b2ab1a72334, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/004997f048964f8e899c92bbb7e20c60, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/4b32645fbde047748112d37fd40bc686, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/b719c2cbd1ac4c1eb5474a80a756bee2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/908d041f31da4dfa89db5bd0447df1cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/bbcf8050b9b44646a530a1d0395b72fd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/201ac72000a346fb9a6b075976f85728, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f10b92b1c4944352b528db2fdf145374, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5ed9ccc0b4d24a30912bf0a2522a1e26, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/05c75617a20748a78806a1708b964392, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/af2fd77c5c0c40568678099735dcdb2a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5c7b827f34ed49c1be6158a76010e00f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f780a4093f774b0389d784df1528c4df, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/95586e36015944509695ebe3ee068fc3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6f5ff36af6d64f248f2d63196d7a594b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/787445c555eb4d40a1360c76243e633c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e91fd9b9df074b5f85371f47517ecc03, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/9a5e5325b8f644288a3dc0b63c1424d4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7e55108592de46c4801f4a3675524629] to archive 2024-11-27T11:35:28,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:35:28,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6705caa12e3f41c387922e628d8293c8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6705caa12e3f41c387922e628d8293c8 2024-11-27T11:35:28,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2465cba93a3f479e93073e61934525d4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2465cba93a3f479e93073e61934525d4 2024-11-27T11:35:28,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7455744b98eb4ab1a5660fa6406d540b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7455744b98eb4ab1a5660fa6406d540b 2024-11-27T11:35:28,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/eaff47fd114e4f77bfe3c7ee676632e3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/eaff47fd114e4f77bfe3c7ee676632e3 2024-11-27T11:35:28,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/a4a56ef44bd14d52b1c6ca47e2358e7d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/a4a56ef44bd14d52b1c6ca47e2358e7d 2024-11-27T11:35:28,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/270cd73c51f04ad989c5d6c7dd3e28c1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/270cd73c51f04ad989c5d6c7dd3e28c1 2024-11-27T11:35:28,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/fe78dd5902684afdac852d315800f38b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/fe78dd5902684afdac852d315800f38b 2024-11-27T11:35:28,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/17da9b666b84417c90263eba2c4d179a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/17da9b666b84417c90263eba2c4d179a 2024-11-27T11:35:28,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/019ea283a1e34bc48879dc6d87467288 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/019ea283a1e34bc48879dc6d87467288 2024-11-27T11:35:28,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/1acaa3fe7ae445328be87f86a32377fb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/1acaa3fe7ae445328be87f86a32377fb 2024-11-27T11:35:28,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f1b9f029be364fc9abf2d204fea4db0a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f1b9f029be364fc9abf2d204fea4db0a 2024-11-27T11:35:28,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e8f8c1e6b4334777911fffe06dbafa02 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e8f8c1e6b4334777911fffe06dbafa02 2024-11-27T11:35:28,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2da3c5ed04cd46b686e040ce46e33b97 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2da3c5ed04cd46b686e040ce46e33b97 2024-11-27T11:35:28,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2865e16edfc94b84a893a45b966458d8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/2865e16edfc94b84a893a45b966458d8 2024-11-27T11:35:28,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/c851fca7a6b44baaa1a00b2ab1a72334 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/c851fca7a6b44baaa1a00b2ab1a72334 2024-11-27T11:35:28,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/004997f048964f8e899c92bbb7e20c60 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/004997f048964f8e899c92bbb7e20c60 2024-11-27T11:35:28,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/4b32645fbde047748112d37fd40bc686 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/4b32645fbde047748112d37fd40bc686 2024-11-27T11:35:28,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/b719c2cbd1ac4c1eb5474a80a756bee2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/b719c2cbd1ac4c1eb5474a80a756bee2 2024-11-27T11:35:28,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/908d041f31da4dfa89db5bd0447df1cd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/908d041f31da4dfa89db5bd0447df1cd 2024-11-27T11:35:28,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/bbcf8050b9b44646a530a1d0395b72fd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/bbcf8050b9b44646a530a1d0395b72fd 2024-11-27T11:35:28,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/201ac72000a346fb9a6b075976f85728 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/201ac72000a346fb9a6b075976f85728 2024-11-27T11:35:28,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f10b92b1c4944352b528db2fdf145374 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f10b92b1c4944352b528db2fdf145374 2024-11-27T11:35:28,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5ed9ccc0b4d24a30912bf0a2522a1e26 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5ed9ccc0b4d24a30912bf0a2522a1e26 2024-11-27T11:35:28,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/05c75617a20748a78806a1708b964392 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/05c75617a20748a78806a1708b964392 2024-11-27T11:35:28,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/af2fd77c5c0c40568678099735dcdb2a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/af2fd77c5c0c40568678099735dcdb2a 2024-11-27T11:35:28,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5c7b827f34ed49c1be6158a76010e00f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/5c7b827f34ed49c1be6158a76010e00f 2024-11-27T11:35:28,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f780a4093f774b0389d784df1528c4df to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f780a4093f774b0389d784df1528c4df 2024-11-27T11:35:28,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/95586e36015944509695ebe3ee068fc3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/95586e36015944509695ebe3ee068fc3 2024-11-27T11:35:28,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6f5ff36af6d64f248f2d63196d7a594b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/6f5ff36af6d64f248f2d63196d7a594b 2024-11-27T11:35:28,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/787445c555eb4d40a1360c76243e633c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/787445c555eb4d40a1360c76243e633c 2024-11-27T11:35:28,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e91fd9b9df074b5f85371f47517ecc03 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/e91fd9b9df074b5f85371f47517ecc03 2024-11-27T11:35:28,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/9a5e5325b8f644288a3dc0b63c1424d4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/9a5e5325b8f644288a3dc0b63c1424d4 2024-11-27T11:35:28,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7e55108592de46c4801f4a3675524629 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/7e55108592de46c4801f4a3675524629 2024-11-27T11:35:28,063 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-27T11:35:28,064 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. 2024-11-27T11:35:28,064 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 080ce2238710db49165afb3a1a396a6b: Waiting for close lock at 1732707326668Running coprocessor pre-close hooks at 1732707326668Disabling compacts and flushes for region at 1732707326668Disabling writes for close at 1732707326669 (+1 ms)Obtaining lock to block concurrent updates at 1732707326669Preparing flush snapshotting stores in 080ce2238710db49165afb3a1a396a6b at 1732707326669Finished memstore snapshotting TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732707326669Flushing stores of TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b. at 1732707326670 (+1 ms)Flushing 080ce2238710db49165afb3a1a396a6b/A: creating writer at 1732707326670Flushing 080ce2238710db49165afb3a1a396a6b/A: appending metadata at 1732707326673 (+3 ms)Flushing 080ce2238710db49165afb3a1a396a6b/A: closing flushed file at 1732707326673Flushing 080ce2238710db49165afb3a1a396a6b/B: creating writer at 1732707327082 (+409 ms)Flushing 080ce2238710db49165afb3a1a396a6b/B: appending metadata at 1732707327085 (+3 ms)Flushing 080ce2238710db49165afb3a1a396a6b/B: closing flushed file at 1732707327085Flushing 080ce2238710db49165afb3a1a396a6b/C: creating writer at 1732707327494 (+409 ms)Flushing 080ce2238710db49165afb3a1a396a6b/C: appending metadata at 1732707327496 (+2 ms)Flushing 080ce2238710db49165afb3a1a396a6b/C: closing flushed file at 1732707327496Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3864c36b: reopening flushed file at 1732707327905 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33c578fd: reopening flushed file at 1732707327910 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66ab3e4f: reopening flushed file at 1732707327915 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 080ce2238710db49165afb3a1a396a6b in 1252ms, sequenceid=495, compaction requested=true at 1732707327921 (+6 ms)Writing region close event to WAL at 1732707328060 (+139 ms)Running coprocessor post-close hooks at 1732707328064 (+4 ms)Closed at 1732707328064 2024-11-27T11:35:28,067 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:28,067 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=080ce2238710db49165afb3a1a396a6b, regionState=CLOSED 2024-11-27T11:35:28,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:35:28,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-27T11:35:28,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 080ce2238710db49165afb3a1a396a6b, server=5b617c7edc8e,35461,1732707299005 in 1.5590 sec 2024-11-27T11:35:28,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-27T11:35:28,074 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=080ce2238710db49165afb3a1a396a6b, UNASSIGN in 1.5660 sec 2024-11-27T11:35:28,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-27T11:35:28,076 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5750 sec 2024-11-27T11:35:28,078 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707328077"}]},"ts":"1732707328077"} 2024-11-27T11:35:28,079 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T11:35:28,079 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T11:35:28,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5890 sec 2024-11-27T11:35:28,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-27T11:35:28,642 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:28,645 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T11:35:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,650 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T11:35:28,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,656 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:28,659 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/recovered.edits] 2024-11-27T11:35:28,662 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4a0c87bbd8d1460586ff1822cefdce2a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4a0c87bbd8d1460586ff1822cefdce2a 2024-11-27T11:35:28,663 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4cc39de433254550ab8bf464d6061230 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/4cc39de433254550ab8bf464d6061230 2024-11-27T11:35:28,664 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d6e33e03058445a684ff064e98519daa to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/A/d6e33e03058445a684ff064e98519daa 2024-11-27T11:35:28,667 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/2d8f104c1c474f25a7d3cb660ad90c20 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/2d8f104c1c474f25a7d3cb660ad90c20 2024-11-27T11:35:28,668 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5ea3b84610a44b0caec751dab5dcadf9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/5ea3b84610a44b0caec751dab5dcadf9 2024-11-27T11:35:28,669 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d0686fecc0254271b2513458fc572eff to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/B/d0686fecc0254271b2513458fc572eff 2024-11-27T11:35:28,672 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/3222f0e794074155b98147a41dd02e30 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/3222f0e794074155b98147a41dd02e30 2024-11-27T11:35:28,673 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/78324f30c3604d73a8f1fd8443df14ca to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/78324f30c3604d73a8f1fd8443df14ca 2024-11-27T11:35:28,674 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f6670f143d9b45539b477993d87b7491 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/C/f6670f143d9b45539b477993d87b7491 2024-11-27T11:35:28,677 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/recovered.edits/498.seqid to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b/recovered.edits/498.seqid 2024-11-27T11:35:28,677 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/080ce2238710db49165afb3a1a396a6b 2024-11-27T11:35:28,677 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T11:35:28,682 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-27T11:35:28,691 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T11:35:28,705 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T11:35:28,721 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T11:35:28,722 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,723 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T11:35:28,723 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732707328723"}]},"ts":"9223372036854775807"} 2024-11-27T11:35:28,730 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T11:35:28,730 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 080ce2238710db49165afb3a1a396a6b, NAME => 'TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T11:35:28,731 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T11:35:28,731 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732707328731"}]},"ts":"9223372036854775807"} 2024-11-27T11:35:28,734 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T11:35:28,735 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 89 msec 2024-11-27T11:35:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-27T11:35:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T11:35:28,764 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:28,783 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1287702838_22 at /127.0.0.1:33968 [Waiting for operation #333] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;5b617c7edc8e:35461-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_612182728_22 at /127.0.0.1:58618 [Waiting for operation #351] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=327 (was 237) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6544 (was 7102) 2024-11-27T11:35:28,793 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=327, ProcessCount=11, AvailableMemoryMB=6541 2024-11-27T11:35:28,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:35:28,795 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:35:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:28,797 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T11:35:28,797 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:28,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-27T11:35:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T11:35:28,798 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T11:35:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741947_1123 (size=959) 2024-11-27T11:35:28,806 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:35:28,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741948_1124 (size=53) 2024-11-27T11:35:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T11:35:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T11:35:29,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:29,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bfc88b634f4291f702daaa0b78cc4f59, disabling compactions & flushes 2024-11-27T11:35:29,212 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. after waiting 0 ms 2024-11-27T11:35:29,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,212 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bfc88b634f4291f702daaa0b78cc4f59: Waiting for close lock at 1732707329212Disabling compacts and flushes for region at 1732707329212Disabling writes for close at 1732707329212Writing region close event to WAL at 1732707329212Closed at 1732707329212 2024-11-27T11:35:29,213 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T11:35:29,214 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732707329214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732707329214"}]},"ts":"1732707329214"} 2024-11-27T11:35:29,215 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T11:35:29,216 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T11:35:29,216 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707329216"}]},"ts":"1732707329216"} 2024-11-27T11:35:29,218 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T11:35:29,218 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, ASSIGN}] 2024-11-27T11:35:29,219 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, ASSIGN 2024-11-27T11:35:29,220 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, ASSIGN; state=OFFLINE, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=false 2024-11-27T11:35:29,371 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:29,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, ASSIGN because future has completed 2024-11-27T11:35:29,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T11:35:29,528 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,528 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:35:29,529 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,529 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:29,529 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,529 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,530 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,532 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfc88b634f4291f702daaa0b78cc4f59 columnFamilyName A 2024-11-27T11:35:29,532 DEBUG [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:29,533 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(327): Store=bfc88b634f4291f702daaa0b78cc4f59/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:29,533 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,534 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfc88b634f4291f702daaa0b78cc4f59 columnFamilyName B 2024-11-27T11:35:29,534 DEBUG [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:29,534 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(327): Store=bfc88b634f4291f702daaa0b78cc4f59/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:29,535 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,536 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfc88b634f4291f702daaa0b78cc4f59 columnFamilyName C 2024-11-27T11:35:29,536 DEBUG [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:29,536 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(327): Store=bfc88b634f4291f702daaa0b78cc4f59/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:29,536 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,538 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,538 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,538 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:35:29,540 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,541 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:35:29,542 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened bfc88b634f4291f702daaa0b78cc4f59; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60772424, jitterRate=-0.09442031383514404}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:35:29,542 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:29,543 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for bfc88b634f4291f702daaa0b78cc4f59: Running coprocessor pre-open hook at 1732707329529Writing region info on filesystem at 1732707329529Initializing all the Stores at 1732707329530 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707329530Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707329530Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707329530Cleaning up temporary data from old regions at 1732707329538 (+8 ms)Running coprocessor post-open hooks at 1732707329542 (+4 ms)Region opened successfully at 1732707329543 (+1 ms) 2024-11-27T11:35:29,544 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., pid=32, masterSystemTime=1732707329525 2024-11-27T11:35:29,545 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,546 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:29,546 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:29,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:35:29,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-27T11:35:29,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 in 175 msec 2024-11-27T11:35:29,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-27T11:35:29,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, ASSIGN in 332 msec 2024-11-27T11:35:29,553 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T11:35:29,553 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707329553"}]},"ts":"1732707329553"} 2024-11-27T11:35:29,554 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T11:35:29,555 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T11:35:29,556 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 760 msec 2024-11-27T11:35:29,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-27T11:35:29,933 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:29,934 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78205da9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:29,934 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:29,934 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:29,935 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:29,935 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f9e7e14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:29,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:29,937 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:29,938 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f289009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:29,938 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:29,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:29,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:29,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54464, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:29,944 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:35:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:35:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@281d24f1 2024-11-27T11:35:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:35:29,946 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41504, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:35:29,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:35:29,952 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:35:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:29,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741949_1125 (size=995) 2024-11-27T11:35:30,371 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T11:35:30,371 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T11:35:30,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:35:30,380 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, REOPEN/MOVE}] 2024-11-27T11:35:30,381 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, REOPEN/MOVE 2024-11-27T11:35:30,382 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:30,383 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, REOPEN/MOVE because future has completed 2024-11-27T11:35:30,383 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:35:30,384 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:30,536 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,536 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:35:30,536 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing bfc88b634f4291f702daaa0b78cc4f59, disabling compactions & flushes 2024-11-27T11:35:30,536 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,536 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,536 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. after waiting 0 ms 2024-11-27T11:35:30,536 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,541 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T11:35:30,541 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,541 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for bfc88b634f4291f702daaa0b78cc4f59: Waiting for close lock at 1732707330536Running coprocessor pre-close hooks at 1732707330536Disabling compacts and flushes for region at 1732707330536Disabling writes for close at 1732707330536Writing region close event to WAL at 1732707330537 (+1 ms)Running coprocessor post-close hooks at 1732707330541 (+4 ms)Closed at 1732707330541 2024-11-27T11:35:30,541 WARN [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: bfc88b634f4291f702daaa0b78cc4f59 to self. 2024-11-27T11:35:30,543 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,544 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=CLOSED 2024-11-27T11:35:30,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:35:30,547 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-27T11:35:30,548 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 in 163 msec 2024-11-27T11:35:30,548 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, REOPEN/MOVE; state=CLOSED, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=true 2024-11-27T11:35:30,699 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:30,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, REOPEN/MOVE because future has completed 2024-11-27T11:35:30,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:30,856 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,856 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:35:30,856 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,856 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:35:30,856 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,856 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,861 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,866 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfc88b634f4291f702daaa0b78cc4f59 columnFamilyName A 2024-11-27T11:35:30,868 DEBUG [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:30,869 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(327): Store=bfc88b634f4291f702daaa0b78cc4f59/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:30,869 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,870 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfc88b634f4291f702daaa0b78cc4f59 columnFamilyName B 2024-11-27T11:35:30,870 DEBUG [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:30,870 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(327): Store=bfc88b634f4291f702daaa0b78cc4f59/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:30,871 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,871 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfc88b634f4291f702daaa0b78cc4f59 columnFamilyName C 2024-11-27T11:35:30,871 DEBUG [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:30,872 INFO [StoreOpener-bfc88b634f4291f702daaa0b78cc4f59-1 {}] regionserver.HStore(327): Store=bfc88b634f4291f702daaa0b78cc4f59/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:35:30,872 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,872 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,873 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,874 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,874 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,875 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:35:30,876 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,877 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened bfc88b634f4291f702daaa0b78cc4f59; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75375180, jitterRate=0.12317770719528198}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:35:30,877 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,877 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for bfc88b634f4291f702daaa0b78cc4f59: Running coprocessor pre-open hook at 1732707330857Writing region info on filesystem at 1732707330857Initializing all the Stores at 1732707330857Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707330857Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707330858 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707330858Cleaning up temporary data from old regions at 1732707330874 (+16 ms)Running coprocessor post-open hooks at 1732707330877 (+3 ms)Region opened successfully at 1732707330877 2024-11-27T11:35:30,878 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., pid=37, masterSystemTime=1732707330852 2024-11-27T11:35:30,880 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,880 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:30,880 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=OPEN, openSeqNum=5, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:30,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:35:30,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-27T11:35:30,884 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 in 181 msec 2024-11-27T11:35:30,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-27T11:35:30,885 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, REOPEN/MOVE in 504 msec 2024-11-27T11:35:30,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-27T11:35:30,888 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 512 msec 2024-11-27T11:35:30,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 934 msec 2024-11-27T11:35:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-27T11:35:30,892 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:30,893 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd952c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,893 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,893 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,894 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d0b098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,896 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e204e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf7761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,897 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,898 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,898 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2149b6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,900 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58aaf524, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,901 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@502d4ad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,901 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,901 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,902 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66569dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,903 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46214, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c6a4b8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,904 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@465441fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,904 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,904 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,905 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,905 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe46890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,907 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@265bbc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,908 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3702b926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,908 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,909 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e214039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,910 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,911 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fcd60a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,912 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f423292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,912 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,912 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,913 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d3cb544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,914 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799077ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,915 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24bb3dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,915 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,916 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,916 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2397fb47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,918 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3eeec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,919 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f988e58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,919 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,919 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,920 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d643a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,921 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5784fa65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,923 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655bb30f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,923 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:35:30,923 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:35:30,923 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:35:30,923 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:35:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:35:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7511142b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:35:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:35:30,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:30,925 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:35:30,925 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48881205, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:35:30,926 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,926 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:30,926 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,926 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,927 DEBUG [Thread-632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,927 DEBUG [Thread-636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,927 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,928 DEBUG [Thread-645 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-27T11:35:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,928 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,929 DEBUG [Thread-638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,929 DEBUG [Thread-645 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T11:35:30,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,930 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,930 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,931 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:30,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57754, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57792, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,932 DEBUG [Thread-634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,932 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:30,932 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:30,932 DEBUG [Thread-630 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:35:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:35:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:35:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,935 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:35:30,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:35:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:30,950 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:30,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:30,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279a075929f9654a89ae5a11d3a100d9a1_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707330949/Put/seqid=0 2024-11-27T11:35:31,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707390988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707390991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707390992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,008 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707391002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707391005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741950_1126 (size=17034) 2024-11-27T11:35:31,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T11:35:31,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:31,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707391113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,118 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707391113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707391113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707391123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707391123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:31,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T11:35:31,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707391322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707391323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707391333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707391333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707391333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:31,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,430 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:31,437 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279a075929f9654a89ae5a11d3a100d9a1_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279a075929f9654a89ae5a11d3a100d9a1_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:31,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6fb0d9df9c094f3ca3b81c05226e7e49, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:31,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6fb0d9df9c094f3ca3b81c05226e7e49 is 175, key is test_row_0/A:col10/1732707330949/Put/seqid=0 2024-11-27T11:35:31,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741951_1127 (size=48139) 2024-11-27T11:35:31,463 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6fb0d9df9c094f3ca3b81c05226e7e49 2024-11-27T11:35:31,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/177f9d73e283443387c4927ba4247e2f is 50, key is test_row_0/B:col10/1732707330949/Put/seqid=0 2024-11-27T11:35:31,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741952_1128 (size=12001) 2024-11-27T11:35:31,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:31,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T11:35:31,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707391633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707391643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707391643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707391643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:31,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:31,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:31,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:31,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:31,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:31,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/177f9d73e283443387c4927ba4247e2f 2024-11-27T11:35:31,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/748c8f56f9ba45d5b136766bcaaa56d2 is 50, key is test_row_0/C:col10/1732707330949/Put/seqid=0 2024-11-27T11:35:31,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741953_1129 (size=12001) 2024-11-27T11:35:31,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/748c8f56f9ba45d5b136766bcaaa56d2 2024-11-27T11:35:31,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6fb0d9df9c094f3ca3b81c05226e7e49 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49 2024-11-27T11:35:31,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49, entries=250, sequenceid=15, filesize=47.0 K 2024-11-27T11:35:31,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/177f9d73e283443387c4927ba4247e2f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/177f9d73e283443387c4927ba4247e2f 2024-11-27T11:35:31,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/177f9d73e283443387c4927ba4247e2f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-27T11:35:31,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/748c8f56f9ba45d5b136766bcaaa56d2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/748c8f56f9ba45d5b136766bcaaa56d2 2024-11-27T11:35:31,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/748c8f56f9ba45d5b136766bcaaa56d2, entries=150, sequenceid=15, filesize=11.7 K 2024-11-27T11:35:31,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bfc88b634f4291f702daaa0b78cc4f59 in 1025ms, sequenceid=15, compaction requested=false 2024-11-27T11:35:31,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:32,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-27T11:35:32,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:32,009 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:35:32,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271921e8c359c14dce8e34f64be5852c55_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707330990/Put/seqid=0 2024-11-27T11:35:32,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741954_1130 (size=12154) 2024-11-27T11:35:32,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:32,046 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271921e8c359c14dce8e34f64be5852c55_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271921e8c359c14dce8e34f64be5852c55_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cd3845958ffb4b359c6012b3c27b7584, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:32,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cd3845958ffb4b359c6012b3c27b7584 is 175, key is test_row_0/A:col10/1732707330990/Put/seqid=0 2024-11-27T11:35:32,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T11:35:32,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741955_1131 (size=30955) 2024-11-27T11:35:32,066 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cd3845958ffb4b359c6012b3c27b7584 2024-11-27T11:35:32,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/5086b025b9df4d8189612359db78b215 is 50, key is test_row_0/B:col10/1732707330990/Put/seqid=0 2024-11-27T11:35:32,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741956_1132 (size=12001) 2024-11-27T11:35:32,097 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/5086b025b9df4d8189612359db78b215 2024-11-27T11:35:32,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/00db576d4f824b5facfd3538442d067a is 50, key is test_row_0/C:col10/1732707330990/Put/seqid=0 2024-11-27T11:35:32,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741957_1133 (size=12001) 2024-11-27T11:35:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:32,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707392163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707392163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707392163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707392163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707392165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707392282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707392282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,287 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707392492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707392492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,543 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/00db576d4f824b5facfd3538442d067a 2024-11-27T11:35:32,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cd3845958ffb4b359c6012b3c27b7584 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584 2024-11-27T11:35:32,554 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T11:35:32,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/5086b025b9df4d8189612359db78b215 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/5086b025b9df4d8189612359db78b215 2024-11-27T11:35:32,560 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/5086b025b9df4d8189612359db78b215, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:35:32,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/00db576d4f824b5facfd3538442d067a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/00db576d4f824b5facfd3538442d067a 2024-11-27T11:35:32,568 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/00db576d4f824b5facfd3538442d067a, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:35:32,571 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bfc88b634f4291f702daaa0b78cc4f59 in 562ms, sequenceid=40, compaction requested=false 2024-11-27T11:35:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:32,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-27T11:35:32,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-27T11:35:32,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-27T11:35:32,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6410 sec 2024-11-27T11:35:32,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.6500 sec 2024-11-27T11:35:32,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:32,806 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:32,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:35:32,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277c0681439dcd4dabbab83d56d0712fc9_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707332163/Put/seqid=0 2024-11-27T11:35:32,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741958_1134 (size=17034) 2024-11-27T11:35:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707392873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707392873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707392983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707392983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:32,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:32,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-27T11:35:33,073 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:33,074 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:33,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-27T11:35:33,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T11:35:33,079 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:33,080 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:33,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:33,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T11:35:33,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707393183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707393183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707393183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,185 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:33,185 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,186 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,186 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707393193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707393193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,198 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T11:35:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:33,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,235 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411277c0681439dcd4dabbab83d56d0712fc9_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c0681439dcd4dabbab83d56d0712fc9_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:33,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/864a04047c1c41a1b00417d753c978ab, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:33,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/864a04047c1c41a1b00417d753c978ab is 175, key is test_row_0/A:col10/1732707332163/Put/seqid=0 2024-11-27T11:35:33,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741959_1135 (size=48139) 2024-11-27T11:35:33,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T11:35:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T11:35:33,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707393503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707393503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:33,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:33,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T11:35:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,599 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:35:33,649 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/864a04047c1c41a1b00417d753c978ab 2024-11-27T11:35:33,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/b30f9fe6ad284563a92d391bf5d48e54 is 50, key is test_row_0/B:col10/1732707332163/Put/seqid=0 2024-11-27T11:35:33,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741960_1136 (size=12001) 2024-11-27T11:35:33,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/b30f9fe6ad284563a92d391bf5d48e54 2024-11-27T11:35:33,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/20140bbeabf04269b9806cf14283cc76 is 50, key is test_row_0/C:col10/1732707332163/Put/seqid=0 2024-11-27T11:35:33,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741961_1137 (size=12001) 2024-11-27T11:35:33,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/20140bbeabf04269b9806cf14283cc76 2024-11-27T11:35:33,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/864a04047c1c41a1b00417d753c978ab as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab 2024-11-27T11:35:33,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab, entries=250, sequenceid=53, filesize=47.0 K 2024-11-27T11:35:33,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T11:35:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:33,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:33,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/b30f9fe6ad284563a92d391bf5d48e54 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/b30f9fe6ad284563a92d391bf5d48e54 2024-11-27T11:35:33,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/b30f9fe6ad284563a92d391bf5d48e54, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T11:35:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T11:35:33,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/20140bbeabf04269b9806cf14283cc76 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/20140bbeabf04269b9806cf14283cc76 2024-11-27T11:35:33,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/20140bbeabf04269b9806cf14283cc76, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T11:35:33,712 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for bfc88b634f4291f702daaa0b78cc4f59 in 906ms, sequenceid=53, compaction requested=true 2024-11-27T11:35:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:33,713 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:33,713 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:33,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:33,715 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:33,715 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:33,715 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:33,715 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:33,715 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,715 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,715 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=124.3 K 2024-11-27T11:35:33,715 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/177f9d73e283443387c4927ba4247e2f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/5086b025b9df4d8189612359db78b215, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/b30f9fe6ad284563a92d391bf5d48e54] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=35.2 K 2024-11-27T11:35:33,716 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,716 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab] 2024-11-27T11:35:33,716 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 177f9d73e283443387c4927ba4247e2f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732707330943 2024-11-27T11:35:33,719 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5086b025b9df4d8189612359db78b215, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707330990 2024-11-27T11:35:33,720 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fb0d9df9c094f3ca3b81c05226e7e49, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732707330941 2024-11-27T11:35:33,720 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b30f9fe6ad284563a92d391bf5d48e54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707332149 2024-11-27T11:35:33,721 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd3845958ffb4b359c6012b3c27b7584, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707330990 2024-11-27T11:35:33,721 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 864a04047c1c41a1b00417d753c978ab, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707332149 2024-11-27T11:35:33,738 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:33,739 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/be6fe9c09fbe4aa2be43d292ed94febc is 50, key is test_row_0/B:col10/1732707332163/Put/seqid=0 2024-11-27T11:35:33,750 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:33,761 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127f4917b4f30e745339ca9f6425bb679e9_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:33,769 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127f4917b4f30e745339ca9f6425bb679e9_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:33,770 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f4917b4f30e745339ca9f6425bb679e9_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:33,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741962_1138 (size=12104) 2024-11-27T11:35:33,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741963_1139 (size=4469) 2024-11-27T11:35:33,792 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#120 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:33,792 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/abaaace8a0c445fe9be257e37001d1d6 is 175, key is test_row_0/A:col10/1732707332163/Put/seqid=0 2024-11-27T11:35:33,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741964_1140 (size=31058) 2024-11-27T11:35:33,848 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-27T11:35:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,849 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:35:33,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112783dc026f3b2043aca2d85ad9f8b6a1c5_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707332871/Put/seqid=0 2024-11-27T11:35:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741965_1141 (size=12154) 2024-11-27T11:35:33,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,889 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112783dc026f3b2043aca2d85ad9f8b6a1c5_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783dc026f3b2043aca2d85ad9f8b6a1c5_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:33,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e6242b8d6f2d4aea9444ed713fe79e89, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:33,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e6242b8d6f2d4aea9444ed713fe79e89 is 175, key is test_row_0/A:col10/1732707332871/Put/seqid=0 2024-11-27T11:35:33,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741966_1142 (size=30955) 2024-11-27T11:35:33,905 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e6242b8d6f2d4aea9444ed713fe79e89 2024-11-27T11:35:33,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/411ab17cbe464f4f8d059b148321017e is 50, key is test_row_0/B:col10/1732707332871/Put/seqid=0 2024-11-27T11:35:33,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741967_1143 (size=12001) 2024-11-27T11:35:33,933 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/411ab17cbe464f4f8d059b148321017e 2024-11-27T11:35:33,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/88bd743566364b49b412e0a8b7b605b4 is 50, key is test_row_0/C:col10/1732707332871/Put/seqid=0 2024-11-27T11:35:33,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741968_1144 (size=12001) 2024-11-27T11:35:33,960 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/88bd743566364b49b412e0a8b7b605b4 2024-11-27T11:35:33,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e6242b8d6f2d4aea9444ed713fe79e89 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89 2024-11-27T11:35:33,973 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T11:35:33,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/411ab17cbe464f4f8d059b148321017e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/411ab17cbe464f4f8d059b148321017e 2024-11-27T11:35:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,980 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/411ab17cbe464f4f8d059b148321017e, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T11:35:33,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/88bd743566364b49b412e0a8b7b605b4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/88bd743566364b49b412e0a8b7b605b4 2024-11-27T11:35:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,988 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/88bd743566364b49b412e0a8b7b605b4, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T11:35:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,989 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for bfc88b634f4291f702daaa0b78cc4f59 in 141ms, sequenceid=77, compaction requested=true 2024-11-27T11:35:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-27T11:35:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-27T11:35:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-27T11:35:33,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 912 msec 2024-11-27T11:35:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 922 msec 2024-11-27T11:35:33,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:34,109 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:34,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:34,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127271bc219301d4fbf989e09b700f3a3c1_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707334105/Put/seqid=0 2024-11-27T11:35:34,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741969_1145 (size=17034) 2024-11-27T11:35:34,183 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/be6fe9c09fbe4aa2be43d292ed94febc as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be6fe9c09fbe4aa2be43d292ed94febc 2024-11-27T11:35:34,189 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into be6fe9c09fbe4aa2be43d292ed94febc(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:34,189 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:34,189 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=13, startTime=1732707333713; duration=0sec 2024-11-27T11:35:34,190 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:34,190 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:34,190 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:34,192 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:34,192 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:34,192 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,192 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/748c8f56f9ba45d5b136766bcaaa56d2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/00db576d4f824b5facfd3538442d067a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/20140bbeabf04269b9806cf14283cc76, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/88bd743566364b49b412e0a8b7b605b4] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=46.9 K 2024-11-27T11:35:34,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 748c8f56f9ba45d5b136766bcaaa56d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732707330943 2024-11-27T11:35:34,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 00db576d4f824b5facfd3538442d067a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707330990 2024-11-27T11:35:34,194 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 20140bbeabf04269b9806cf14283cc76, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707332149 2024-11-27T11:35:34,194 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bd743566364b49b412e0a8b7b605b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707332837 2024-11-27T11:35:34,210 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:34,210 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/abaaace8a0c445fe9be257e37001d1d6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/abaaace8a0c445fe9be257e37001d1d6 2024-11-27T11:35:34,211 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/3d8a804094f9456099dc7d2e300d4266 is 50, key is test_row_0/C:col10/1732707332871/Put/seqid=0 2024-11-27T11:35:34,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-27T11:35:34,213 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:34,214 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:34,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-27T11:35:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:34,217 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:34,219 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:34,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:34,220 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into abaaace8a0c445fe9be257e37001d1d6(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:34,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:34,220 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=13, startTime=1732707333713; duration=0sec 2024-11-27T11:35:34,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:34,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:34,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741970_1146 (size=12139) 2024-11-27T11:35:34,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707394222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707394224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,237 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/3d8a804094f9456099dc7d2e300d4266 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/3d8a804094f9456099dc7d2e300d4266 2024-11-27T11:35:34,246 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into 3d8a804094f9456099dc7d2e300d4266(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:34,246 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:34,246 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=12, startTime=1732707333713; duration=0sec 2024-11-27T11:35:34,246 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:34,246 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:34,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:34,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707394333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,335 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707394343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:34,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:34,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707394543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707394553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,554 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,554 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,559 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127271bc219301d4fbf989e09b700f3a3c1_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127271bc219301d4fbf989e09b700f3a3c1_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:34,560 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cbef93e810a346ffbc5a1240f96da2a0, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:34,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cbef93e810a346ffbc5a1240f96da2a0 is 175, key is test_row_0/A:col10/1732707334105/Put/seqid=0 2024-11-27T11:35:34,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741971_1147 (size=48139) 2024-11-27T11:35:34,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707394852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707394863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:34,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=88, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cbef93e810a346ffbc5a1240f96da2a0 2024-11-27T11:35:34,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/3bf749f857894e92b4689f91dc087964 is 50, key is test_row_0/B:col10/1732707334105/Put/seqid=0 2024-11-27T11:35:34,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:34,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:34,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741972_1148 (size=12001) 2024-11-27T11:35:35,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/3bf749f857894e92b4689f91dc087964 2024-11-27T11:35:35,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/4a1dd1f7111f466b858cd98e9e4f6196 is 50, key is test_row_0/C:col10/1732707334105/Put/seqid=0 2024-11-27T11:35:35,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741973_1149 (size=12001) 2024-11-27T11:35:35,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:35,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707395203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:35,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:35,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707395203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:35,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:35,205 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707395203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:35,206 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:35,206 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:35,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:35,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707395363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:35,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707395382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:35,387 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:35,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/4a1dd1f7111f466b858cd98e9e4f6196 2024-11-27T11:35:35,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cbef93e810a346ffbc5a1240f96da2a0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0 2024-11-27T11:35:35,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0, entries=250, sequenceid=88, filesize=47.0 K 2024-11-27T11:35:35,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/3bf749f857894e92b4689f91dc087964 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/3bf749f857894e92b4689f91dc087964 2024-11-27T11:35:35,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/3bf749f857894e92b4689f91dc087964, entries=150, sequenceid=88, filesize=11.7 K 2024-11-27T11:35:35,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/4a1dd1f7111f466b858cd98e9e4f6196 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4a1dd1f7111f466b858cd98e9e4f6196 2024-11-27T11:35:35,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4a1dd1f7111f466b858cd98e9e4f6196, entries=150, sequenceid=88, filesize=11.7 K 2024-11-27T11:35:35,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bfc88b634f4291f702daaa0b78cc4f59 in 1338ms, sequenceid=88, compaction requested=true 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:35,448 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:35,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:35,448 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:35,450 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:35,450 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:35,450 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,450 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/abaaace8a0c445fe9be257e37001d1d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=107.6 K 2024-11-27T11:35:35,450 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,450 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/abaaace8a0c445fe9be257e37001d1d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0] 2024-11-27T11:35:35,450 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:35,450 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:35,450 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,450 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be6fe9c09fbe4aa2be43d292ed94febc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/411ab17cbe464f4f8d059b148321017e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/3bf749f857894e92b4689f91dc087964] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=35.3 K 2024-11-27T11:35:35,451 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting be6fe9c09fbe4aa2be43d292ed94febc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707332149 2024-11-27T11:35:35,453 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting abaaace8a0c445fe9be257e37001d1d6, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707332149 2024-11-27T11:35:35,454 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 411ab17cbe464f4f8d059b148321017e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707332837 2024-11-27T11:35:35,454 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6242b8d6f2d4aea9444ed713fe79e89, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707332837 2024-11-27T11:35:35,454 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bf749f857894e92b4689f91dc087964, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732707334099 2024-11-27T11:35:35,454 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbef93e810a346ffbc5a1240f96da2a0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732707334090 2024-11-27T11:35:35,464 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:35,466 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:35,467 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/8cd8005730bb4eadb0519bbe58fc0099 is 50, key is test_row_0/B:col10/1732707334105/Put/seqid=0 2024-11-27T11:35:35,477 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275819fe857fd54067869c50bbe5cf4984_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:35,479 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275819fe857fd54067869c50bbe5cf4984_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:35,480 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275819fe857fd54067869c50bbe5cf4984_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:35,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741974_1150 (size=12207) 2024-11-27T11:35:35,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741975_1151 (size=4469) 2024-11-27T11:35:35,521 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#128 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:35,521 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/c58a54aaf7f546cf96887d1586facb33 is 175, key is test_row_0/A:col10/1732707334105/Put/seqid=0 2024-11-27T11:35:35,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741976_1152 (size=31161) 2024-11-27T11:35:35,550 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/c58a54aaf7f546cf96887d1586facb33 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/c58a54aaf7f546cf96887d1586facb33 2024-11-27T11:35:35,561 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into c58a54aaf7f546cf96887d1586facb33(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:35,561 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:35,561 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=13, startTime=1732707335448; duration=0sec 2024-11-27T11:35:35,561 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:35,561 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:35,561 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-27T11:35:35,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-27T11:35:35,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-27T11:35:35,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. because compaction request was cancelled 2024-11-27T11:35:35,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:35,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-27T11:35:35,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:35,602 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:35:35,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eceaa2edbc03472e92f6530f2b1ef054_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707334220/Put/seqid=0 2024-11-27T11:35:35,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741977_1153 (size=12154) 2024-11-27T11:35:35,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:35,625 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eceaa2edbc03472e92f6530f2b1ef054_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eceaa2edbc03472e92f6530f2b1ef054_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:35,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9c8fdd35ce5d4da386677fefdcb91d55, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:35,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9c8fdd35ce5d4da386677fefdcb91d55 is 175, key is test_row_0/A:col10/1732707334220/Put/seqid=0 2024-11-27T11:35:35,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741978_1154 (size=30955) 2024-11-27T11:35:35,900 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/8cd8005730bb4eadb0519bbe58fc0099 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8cd8005730bb4eadb0519bbe58fc0099 2024-11-27T11:35:35,906 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into 8cd8005730bb4eadb0519bbe58fc0099(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:35,906 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:35,906 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=13, startTime=1732707335448; duration=0sec 2024-11-27T11:35:35,906 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:35,906 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:36,037 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9c8fdd35ce5d4da386677fefdcb91d55 2024-11-27T11:35:36,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/f6ae478963954a8b84875f6347e50ca2 is 50, key is test_row_0/B:col10/1732707334220/Put/seqid=0 2024-11-27T11:35:36,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741979_1155 (size=12001) 2024-11-27T11:35:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:36,384 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707396406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707396407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,452 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/f6ae478963954a8b84875f6347e50ca2 2024-11-27T11:35:36,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/f20c55261a4844249a7c5387658b74d5 is 50, key is test_row_0/C:col10/1732707334220/Put/seqid=0 2024-11-27T11:35:36,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741980_1156 (size=12001) 2024-11-27T11:35:36,472 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/f20c55261a4844249a7c5387658b74d5 2024-11-27T11:35:36,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9c8fdd35ce5d4da386677fefdcb91d55 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55 2024-11-27T11:35:36,483 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55, entries=150, sequenceid=117, filesize=30.2 K 2024-11-27T11:35:36,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/f6ae478963954a8b84875f6347e50ca2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/f6ae478963954a8b84875f6347e50ca2 2024-11-27T11:35:36,489 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/f6ae478963954a8b84875f6347e50ca2, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T11:35:36,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/f20c55261a4844249a7c5387658b74d5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f20c55261a4844249a7c5387658b74d5 2024-11-27T11:35:36,497 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f20c55261a4844249a7c5387658b74d5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-27T11:35:36,499 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bfc88b634f4291f702daaa0b78cc4f59 in 897ms, sequenceid=117, compaction requested=true 2024-11-27T11:35:36,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:36,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:36,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-27T11:35:36,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-27T11:35:36,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-27T11:35:36,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2820 sec 2024-11-27T11:35:36,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2900 sec 2024-11-27T11:35:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:36,514 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:36,514 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:35:36,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dada4e7eed4241efbe6d38d841625d83_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:36,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741981_1157 (size=14644) 2024-11-27T11:35:36,538 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:36,543 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dada4e7eed4241efbe6d38d841625d83_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dada4e7eed4241efbe6d38d841625d83_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:36,544 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e81723df27f3462b83fe6fa73a25b607, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:36,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e81723df27f3462b83fe6fa73a25b607 is 175, key is test_row_0/A:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:36,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741982_1158 (size=39599) 2024-11-27T11:35:36,554 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e81723df27f3462b83fe6fa73a25b607 2024-11-27T11:35:36,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/54022562d96a4274bf7e9cd4ecdda4d5 is 50, key is test_row_0/B:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:36,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741983_1159 (size=12051) 2024-11-27T11:35:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707396583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707396585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,591 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707396692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707396702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,703 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707396903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:36,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707396912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:36,914 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:36,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/54022562d96a4274bf7e9cd4ecdda4d5 2024-11-27T11:35:36,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/19b5e8b9086e4b088646ae0065de8a59 is 50, key is test_row_0/C:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:36,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741984_1160 (size=12051) 2024-11-27T11:35:37,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707397213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:37,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:37,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707397223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:37,388 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/19b5e8b9086e4b088646ae0065de8a59 2024-11-27T11:35:37,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/e81723df27f3462b83fe6fa73a25b607 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607 2024-11-27T11:35:37,399 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607, entries=200, sequenceid=130, filesize=38.7 K 2024-11-27T11:35:37,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/54022562d96a4274bf7e9cd4ecdda4d5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/54022562d96a4274bf7e9cd4ecdda4d5 2024-11-27T11:35:37,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/54022562d96a4274bf7e9cd4ecdda4d5, entries=150, sequenceid=130, filesize=11.8 K 2024-11-27T11:35:37,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/19b5e8b9086e4b088646ae0065de8a59 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/19b5e8b9086e4b088646ae0065de8a59 2024-11-27T11:35:37,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/19b5e8b9086e4b088646ae0065de8a59, entries=150, sequenceid=130, filesize=11.8 K 2024-11-27T11:35:37,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for bfc88b634f4291f702daaa0b78cc4f59 in 898ms, sequenceid=130, compaction requested=true 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:37,413 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:37,413 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:37,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:37,414 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:37,415 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101715 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:37,415 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:37,415 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:37,415 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:37,415 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/c58a54aaf7f546cf96887d1586facb33, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=99.3 K 2024-11-27T11:35:37,415 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:37,415 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/c58a54aaf7f546cf96887d1586facb33, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607] 2024-11-27T11:35:37,415 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:37,415 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8cd8005730bb4eadb0519bbe58fc0099, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/f6ae478963954a8b84875f6347e50ca2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/54022562d96a4274bf7e9cd4ecdda4d5] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=35.4 K 2024-11-27T11:35:37,416 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c58a54aaf7f546cf96887d1586facb33, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732707334099 2024-11-27T11:35:37,416 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cd8005730bb4eadb0519bbe58fc0099, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732707334099 2024-11-27T11:35:37,416 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c8fdd35ce5d4da386677fefdcb91d55, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732707334198 2024-11-27T11:35:37,416 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f6ae478963954a8b84875f6347e50ca2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732707334198 2024-11-27T11:35:37,416 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e81723df27f3462b83fe6fa73a25b607, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732707336401 2024-11-27T11:35:37,416 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 54022562d96a4274bf7e9cd4ecdda4d5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732707336401 2024-11-27T11:35:37,427 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:37,428 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/6f93082e28694724969f2fc5a0f8507e is 50, key is test_row_0/B:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:37,429 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:37,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741985_1161 (size=12359) 2024-11-27T11:35:37,444 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411275df6fe00d76c4e3d85ee45f86dd9e5c0_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:37,447 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411275df6fe00d76c4e3d85ee45f86dd9e5c0_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:37,447 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411275df6fe00d76c4e3d85ee45f86dd9e5c0_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:37,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741986_1162 (size=4469) 2024-11-27T11:35:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:37,736 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:37,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:35:37,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127801fc237cc004ab4bc8eef0b249b796d_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707337736/Put/seqid=0 2024-11-27T11:35:37,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707397751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:37,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:37,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:37,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741987_1163 (size=17284) 2024-11-27T11:35:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707397754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:37,849 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/6f93082e28694724969f2fc5a0f8507e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6f93082e28694724969f2fc5a0f8507e 2024-11-27T11:35:37,856 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into 6f93082e28694724969f2fc5a0f8507e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:37,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:37,856 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=13, startTime=1732707337413; duration=0sec 2024-11-27T11:35:37,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:37,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:37,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:37,857 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:37,857 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:37,857 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:37,858 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/3d8a804094f9456099dc7d2e300d4266, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4a1dd1f7111f466b858cd98e9e4f6196, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f20c55261a4844249a7c5387658b74d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/19b5e8b9086e4b088646ae0065de8a59] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=47.1 K 2024-11-27T11:35:37,858 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d8a804094f9456099dc7d2e300d4266, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707332837 2024-11-27T11:35:37,859 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a1dd1f7111f466b858cd98e9e4f6196, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1732707334099 2024-11-27T11:35:37,860 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#137 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:37,860 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f20c55261a4844249a7c5387658b74d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732707334198 2024-11-27T11:35:37,861 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/d5dc9fea5bc245e6bd79b170ac9f0846 is 175, key is test_row_0/A:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:37,861 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 19b5e8b9086e4b088646ae0065de8a59, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732707336401 2024-11-27T11:35:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707397863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707397863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:37,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:37,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741988_1164 (size=31313) 2024-11-27T11:35:37,878 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:37,878 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/9522d497dad24e70ac6cbe3bfccd87d1 is 50, key is test_row_0/C:col10/1732707336405/Put/seqid=0 2024-11-27T11:35:37,884 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/d5dc9fea5bc245e6bd79b170ac9f0846 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d5dc9fea5bc245e6bd79b170ac9f0846 2024-11-27T11:35:37,891 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into d5dc9fea5bc245e6bd79b170ac9f0846(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:37,891 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:37,891 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=13, startTime=1732707337413; duration=0sec 2024-11-27T11:35:37,891 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:37,891 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:37,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741989_1165 (size=12325) 2024-11-27T11:35:38,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707398073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:38,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707398073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:38,160 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:38,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127801fc237cc004ab4bc8eef0b249b796d_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127801fc237cc004ab4bc8eef0b249b796d_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:38,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/aba95792b12744549c72f1251840fe35, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:38,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/aba95792b12744549c72f1251840fe35 is 175, key is test_row_0/A:col10/1732707337736/Put/seqid=0 2024-11-27T11:35:38,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741990_1166 (size=48389) 2024-11-27T11:35:38,304 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/9522d497dad24e70ac6cbe3bfccd87d1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9522d497dad24e70ac6cbe3bfccd87d1 2024-11-27T11:35:38,311 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into 9522d497dad24e70ac6cbe3bfccd87d1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:38,311 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:38,311 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=12, startTime=1732707337413; duration=0sec 2024-11-27T11:35:38,311 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:38,311 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:38,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-27T11:35:38,383 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:38,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707398382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:38,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:38,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707398383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:38,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-27T11:35:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:38,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T11:35:38,386 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:38,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:38,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T11:35:38,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T11:35:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:38,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:38,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,572 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/aba95792b12744549c72f1251840fe35 2024-11-27T11:35:38,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/37a540b175a442b4b8238008a8c2beb7 is 50, key is test_row_0/B:col10/1732707337736/Put/seqid=0 2024-11-27T11:35:38,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741991_1167 (size=12151) 2024-11-27T11:35:38,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T11:35:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:38,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:38,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T11:35:38,705 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T11:35:38,705 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-27T11:35:38,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T11:35:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:38,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:38,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707398893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:38,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707398893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:38,896 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:38,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/37a540b175a442b4b8238008a8c2beb7 2024-11-27T11:35:38,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/db1f81f168594040a345a4c4dd5f1b04 is 50, key is test_row_0/C:col10/1732707337736/Put/seqid=0 2024-11-27T11:35:38,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T11:35:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:39,000 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:39,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:39,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:39,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741992_1168 (size=12151) 2024-11-27T11:35:39,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/db1f81f168594040a345a4c4dd5f1b04 2024-11-27T11:35:39,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/aba95792b12744549c72f1251840fe35 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35 2024-11-27T11:35:39,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T11:35:39,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35, entries=250, sequenceid=156, filesize=47.3 K 2024-11-27T11:35:39,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/37a540b175a442b4b8238008a8c2beb7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/37a540b175a442b4b8238008a8c2beb7 2024-11-27T11:35:39,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/37a540b175a442b4b8238008a8c2beb7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T11:35:39,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/db1f81f168594040a345a4c4dd5f1b04 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/db1f81f168594040a345a4c4dd5f1b04 2024-11-27T11:35:39,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/db1f81f168594040a345a4c4dd5f1b04, entries=150, sequenceid=156, filesize=11.9 K 2024-11-27T11:35:39,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for bfc88b634f4291f702daaa0b78cc4f59 in 1295ms, sequenceid=156, compaction requested=false 2024-11-27T11:35:39,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-27T11:35:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:39,153 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T11:35:39,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278403ed9a9b2c440ca1b850eea255d9a0_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707337753/Put/seqid=0 2024-11-27T11:35:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741993_1169 (size=9814) 2024-11-27T11:35:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,191 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278403ed9a9b2c440ca1b850eea255d9a0_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278403ed9a9b2c440ca1b850eea255d9a0_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/529c1bc1591d4ddea34be71d09f7d1aa, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/529c1bc1591d4ddea34be71d09f7d1aa is 175, key is test_row_0/A:col10/1732707337753/Put/seqid=0 2024-11-27T11:35:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741994_1170 (size=22461) 2024-11-27T11:35:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:39,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707399307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707399312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707399313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707399422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707399422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707399423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T11:35:39,601 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/529c1bc1591d4ddea34be71d09f7d1aa 2024-11-27T11:35:39,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/be93484bdb724360837efe30e1867e83 is 50, key is test_row_0/B:col10/1732707337753/Put/seqid=0 2024-11-27T11:35:39,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741995_1171 (size=9757) 2024-11-27T11:35:39,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707399633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707399633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707399635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707399912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707399912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,916 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:39,916 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707399943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707399943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707399943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:39,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,022 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/be93484bdb724360837efe30e1867e83 2024-11-27T11:35:40,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/0b1cd520766b419681b37f351d37df12 is 50, key is test_row_0/C:col10/1732707337753/Put/seqid=0 2024-11-27T11:35:40,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741996_1172 (size=9757) 2024-11-27T11:35:40,037 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/0b1cd520766b419681b37f351d37df12 2024-11-27T11:35:40,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/529c1bc1591d4ddea34be71d09f7d1aa as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa 2024-11-27T11:35:40,047 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa, entries=100, sequenceid=169, filesize=21.9 K 2024-11-27T11:35:40,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/be93484bdb724360837efe30e1867e83 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be93484bdb724360837efe30e1867e83 2024-11-27T11:35:40,054 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be93484bdb724360837efe30e1867e83, entries=100, sequenceid=169, filesize=9.5 K 2024-11-27T11:35:40,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/0b1cd520766b419681b37f351d37df12 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0b1cd520766b419681b37f351d37df12 2024-11-27T11:35:40,061 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0b1cd520766b419681b37f351d37df12, entries=100, sequenceid=169, filesize=9.5 K 2024-11-27T11:35:40,062 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for bfc88b634f4291f702daaa0b78cc4f59 in 908ms, sequenceid=169, compaction requested=true 2024-11-27T11:35:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-27T11:35:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-27T11:35:40,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-27T11:35:40,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6770 sec 2024-11-27T11:35:40,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.6840 sec 2024-11-27T11:35:40,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:40,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:40,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-27T11:35:40,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112729d35976ab464024969a08d26a6bdb38_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:40,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707400463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707400464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707400470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741997_1173 (size=17284) 2024-11-27T11:35:40,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,489 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112729d35976ab464024969a08d26a6bdb38_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112729d35976ab464024969a08d26a6bdb38_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:40,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/3f1b2c137aaf4dd180663675aff6f112, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:40,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/3f1b2c137aaf4dd180663675aff6f112 is 175, key is test_row_0/A:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:40,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741998_1174 (size=48389) 2024-11-27T11:35:40,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/3f1b2c137aaf4dd180663675aff6f112 2024-11-27T11:35:40,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d1326a2e4c544653a4667daebe64bf1c is 50, key is test_row_0/B:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:40,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741999_1175 (size=12151) 2024-11-27T11:35:40,516 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d1326a2e4c544653a4667daebe64bf1c 2024-11-27T11:35:40,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/939f8799b9af4003b19813629d0d3100 is 50, key is test_row_0/C:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:40,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742000_1176 (size=12151) 2024-11-27T11:35:40,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-27T11:35:40,543 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:40,544 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:40,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-27T11:35:40,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T11:35:40,546 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:40,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:40,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707400582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707400582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707400582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T11:35:40,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T11:35:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:40,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707400792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707400792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707400792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:40,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:40,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T11:35:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:40,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:40,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:40,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T11:35:40,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/939f8799b9af4003b19813629d0d3100 2024-11-27T11:35:40,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/3f1b2c137aaf4dd180663675aff6f112 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112 2024-11-27T11:35:40,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112, entries=250, sequenceid=197, filesize=47.3 K 2024-11-27T11:35:40,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d1326a2e4c544653a4667daebe64bf1c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d1326a2e4c544653a4667daebe64bf1c 2024-11-27T11:35:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d1326a2e4c544653a4667daebe64bf1c, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T11:35:40,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/939f8799b9af4003b19813629d0d3100 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/939f8799b9af4003b19813629d0d3100 2024-11-27T11:35:40,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/939f8799b9af4003b19813629d0d3100, entries=150, sequenceid=197, filesize=11.9 K 2024-11-27T11:35:40,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,961 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for bfc88b634f4291f702daaa0b78cc4f59 in 505ms, sequenceid=197, compaction requested=true 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:40,961 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:40,961 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:40,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:40,961 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:40,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,963 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46418 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:40,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,963 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:40,963 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150552 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:40,963 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,963 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:40,963 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,963 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6f93082e28694724969f2fc5a0f8507e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/37a540b175a442b4b8238008a8c2beb7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be93484bdb724360837efe30e1867e83, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d1326a2e4c544653a4667daebe64bf1c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=45.3 K 2024-11-27T11:35:40,963 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d5dc9fea5bc245e6bd79b170ac9f0846, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=147.0 K 2024-11-27T11:35:40,964 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:40,964 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d5dc9fea5bc245e6bd79b170ac9f0846, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112] 2024-11-27T11:35:40,964 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5dc9fea5bc245e6bd79b170ac9f0846, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732707336401 2024-11-27T11:35:40,964 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f93082e28694724969f2fc5a0f8507e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732707336401 2024-11-27T11:35:40,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,965 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a540b175a442b4b8238008a8c2beb7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732707336580 2024-11-27T11:35:40,965 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting aba95792b12744549c72f1251840fe35, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732707336579 2024-11-27T11:35:40,965 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting be93484bdb724360837efe30e1867e83, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732707337745 2024-11-27T11:35:40,965 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529c1bc1591d4ddea34be71d09f7d1aa, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732707337745 2024-11-27T11:35:40,966 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d1326a2e4c544653a4667daebe64bf1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732707339310 2024-11-27T11:35:40,966 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1b2c137aaf4dd180663675aff6f112, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732707339289 2024-11-27T11:35:40,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,980 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:40,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,980 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/7caf977aa7294ffaad210a7d32cd5204 is 50, key is test_row_0/B:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,985 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,990 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127318d111af9d1411392c9fb90efa9e0fd_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,993 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127318d111af9d1411392c9fb90efa9e0fd_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:40,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,993 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127318d111af9d1411392c9fb90efa9e0fd_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742001_1177 (size=12595) 2024-11-27T11:35:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:40,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-27T11:35:41,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:41,008 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T11:35:41,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742002_1178 (size=4469) 2024-11-27T11:35:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,012 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#149 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,013 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9712c9c17b584f52a813cc41b96c9e64 is 175, key is test_row_0/A:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f3a7b084558e4ee0b44abbb410b2905b_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707340469/Put/seqid=0 2024-11-27T11:35:41,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742003_1179 (size=31549) 2024-11-27T11:35:41,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,028 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9712c9c17b584f52a813cc41b96c9e64 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9712c9c17b584f52a813cc41b96c9e64 2024-11-27T11:35:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,035 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into 9712c9c17b584f52a813cc41b96c9e64(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:41,035 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:41,035 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=12, startTime=1732707340961; duration=0sec 2024-11-27T11:35:41,035 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:41,035 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:41,035 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,038 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46384 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:41,039 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:41,039 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:41,039 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9522d497dad24e70ac6cbe3bfccd87d1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/db1f81f168594040a345a4c4dd5f1b04, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0b1cd520766b419681b37f351d37df12, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/939f8799b9af4003b19813629d0d3100] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=45.3 K 2024-11-27T11:35:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,039 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9522d497dad24e70ac6cbe3bfccd87d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732707336401 2024-11-27T11:35:41,040 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting db1f81f168594040a345a4c4dd5f1b04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732707336580 2024-11-27T11:35:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,040 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b1cd520766b419681b37f351d37df12, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732707337745 2024-11-27T11:35:41,041 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 939f8799b9af4003b19813629d0d3100, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732707339310 2024-11-27T11:35:41,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742004_1180 (size=9814) 2024-11-27T11:35:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,057 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#151 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,058 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/586fc0371f21461fbe5cb5e6219d5912 is 50, key is test_row_0/C:col10/1732707339310/Put/seqid=0 2024-11-27T11:35:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742005_1181 (size=12561) 2024-11-27T11:35:41,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:41,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:41,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707401161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707401167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707401167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T11:35:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707401273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707401282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707401282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,402 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/7caf977aa7294ffaad210a7d32cd5204 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/7caf977aa7294ffaad210a7d32cd5204 2024-11-27T11:35:41,411 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into 7caf977aa7294ffaad210a7d32cd5204(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:41,411 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:41,411 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=12, startTime=1732707340961; duration=0sec 2024-11-27T11:35:41,411 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:41,412 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:41,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:41,457 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f3a7b084558e4ee0b44abbb410b2905b_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3a7b084558e4ee0b44abbb410b2905b_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:41,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1fcc5ce7ad2b44c0bd746633c3af8c07, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:41,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1fcc5ce7ad2b44c0bd746633c3af8c07 is 175, key is test_row_0/A:col10/1732707340469/Put/seqid=0 2024-11-27T11:35:41,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742006_1182 (size=22461) 2024-11-27T11:35:41,466 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1fcc5ce7ad2b44c0bd746633c3af8c07 2024-11-27T11:35:41,474 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/586fc0371f21461fbe5cb5e6219d5912 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/586fc0371f21461fbe5cb5e6219d5912 2024-11-27T11:35:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/96f885ea1dc342b0aca87315b1bc1b6d is 50, key is test_row_0/B:col10/1732707340469/Put/seqid=0 2024-11-27T11:35:41,483 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into 586fc0371f21461fbe5cb5e6219d5912(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:41,483 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:41,483 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=12, startTime=1732707340961; duration=0sec 2024-11-27T11:35:41,483 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:41,484 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:41,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742007_1183 (size=9757) 2024-11-27T11:35:41,486 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/96f885ea1dc342b0aca87315b1bc1b6d 2024-11-27T11:35:41,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707401483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707401493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/09fd59351f6d4cc19aa75d2f1070976f is 50, key is test_row_0/C:col10/1732707340469/Put/seqid=0 2024-11-27T11:35:41,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707401493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742008_1184 (size=9757) 2024-11-27T11:35:41,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T11:35:41,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707401793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707401802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707401802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,904 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/09fd59351f6d4cc19aa75d2f1070976f 2024-11-27T11:35:41,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1fcc5ce7ad2b44c0bd746633c3af8c07 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07 2024-11-27T11:35:41,914 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07, entries=100, sequenceid=205, filesize=21.9 K 2024-11-27T11:35:41,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/96f885ea1dc342b0aca87315b1bc1b6d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/96f885ea1dc342b0aca87315b1bc1b6d 2024-11-27T11:35:41,919 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/96f885ea1dc342b0aca87315b1bc1b6d, entries=100, sequenceid=205, filesize=9.5 K 2024-11-27T11:35:41,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/09fd59351f6d4cc19aa75d2f1070976f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/09fd59351f6d4cc19aa75d2f1070976f 2024-11-27T11:35:41,927 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/09fd59351f6d4cc19aa75d2f1070976f, entries=100, sequenceid=205, filesize=9.5 K 2024-11-27T11:35:41,928 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for bfc88b634f4291f702daaa0b78cc4f59 in 921ms, sequenceid=205, compaction requested=false 2024-11-27T11:35:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:41,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-27T11:35:41,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-27T11:35:41,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-27T11:35:41,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3820 sec 2024-11-27T11:35:41,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.3870 sec 2024-11-27T11:35:41,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:41,934 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:41,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-27T11:35:41,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707401938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:41,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707401940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:41,942 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:41,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127210cdd5e0a3e42a0b573246aea1100ce_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:41,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742009_1185 (size=17284) 2024-11-27T11:35:42,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707402042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,043 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,044 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707402052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707402252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707402262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,263 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707402303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707402312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707402312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,316 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,350 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:42,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127210cdd5e0a3e42a0b573246aea1100ce_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127210cdd5e0a3e42a0b573246aea1100ce_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:42,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/71cb98c703d04a4e841c5bd23c68b81e, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:42,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/71cb98c703d04a4e841c5bd23c68b81e is 175, key is test_row_0/A:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:42,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742010_1186 (size=48389) 2024-11-27T11:35:42,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707402562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,565 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:42,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707402572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:42,575 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-27T11:35:42,693 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:42,693 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-27T11:35:42,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T11:35:42,695 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:42,696 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:42,696 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:42,762 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/71cb98c703d04a4e841c5bd23c68b81e 2024-11-27T11:35:42,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/8102d3db7a83432a913302af33b6363a is 50, key is test_row_0/B:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:42,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742011_1187 (size=12151) 2024-11-27T11:35:42,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/8102d3db7a83432a913302af33b6363a 2024-11-27T11:35:42,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/9609a401f209465cb18d83540cd04edd is 50, key is test_row_0/C:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:42,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742012_1188 (size=12151) 2024-11-27T11:35:42,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/9609a401f209465cb18d83540cd04edd 2024-11-27T11:35:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T11:35:42,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/71cb98c703d04a4e841c5bd23c68b81e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e 2024-11-27T11:35:42,813 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e, entries=250, sequenceid=238, filesize=47.3 K 2024-11-27T11:35:42,814 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/8102d3db7a83432a913302af33b6363a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8102d3db7a83432a913302af33b6363a 2024-11-27T11:35:42,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8102d3db7a83432a913302af33b6363a, entries=150, sequenceid=238, filesize=11.9 K 2024-11-27T11:35:42,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/9609a401f209465cb18d83540cd04edd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9609a401f209465cb18d83540cd04edd 2024-11-27T11:35:42,825 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9609a401f209465cb18d83540cd04edd, entries=150, sequenceid=238, filesize=11.9 K 2024-11-27T11:35:42,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=20.13 KB/20610 for bfc88b634f4291f702daaa0b78cc4f59 in 892ms, sequenceid=238, compaction requested=true 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:42,826 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:42,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:42,826 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:42,827 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:42,827 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34503 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:42,827 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:42,827 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:42,828 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:42,828 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:42,828 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/7caf977aa7294ffaad210a7d32cd5204, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/96f885ea1dc342b0aca87315b1bc1b6d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8102d3db7a83432a913302af33b6363a] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=33.7 K 2024-11-27T11:35:42,828 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7caf977aa7294ffaad210a7d32cd5204, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732707339310 2024-11-27T11:35:42,828 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f885ea1dc342b0aca87315b1bc1b6d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732707340460 2024-11-27T11:35:42,829 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9712c9c17b584f52a813cc41b96c9e64, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=100.0 K 2024-11-27T11:35:42,829 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:42,829 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9712c9c17b584f52a813cc41b96c9e64, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e] 2024-11-27T11:35:42,829 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8102d3db7a83432a913302af33b6363a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732707341155 2024-11-27T11:35:42,829 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9712c9c17b584f52a813cc41b96c9e64, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732707339310 2024-11-27T11:35:42,830 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1fcc5ce7ad2b44c0bd746633c3af8c07, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732707340460 2024-11-27T11:35:42,831 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71cb98c703d04a4e841c5bd23c68b81e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732707341153 2024-11-27T11:35:42,839 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:42,841 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:42,842 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/ed008587b92246528346656cdc682a9c is 50, key is test_row_0/B:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:42,844 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c0fca8d3182c44939a33c2a1c7f63692_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:42,847 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c0fca8d3182c44939a33c2a1c7f63692_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:42,847 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c0fca8d3182c44939a33c2a1c7f63692_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:42,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-27T11:35:42,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:42,849 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-27T11:35:42,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742013_1189 (size=12697) 2024-11-27T11:35:42,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742014_1190 (size=4469) 2024-11-27T11:35:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112767b127815f3e448eb68bdc65ff01dfb4_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707341937/Put/seqid=0 2024-11-27T11:35:42,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742015_1191 (size=12304) 2024-11-27T11:35:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T11:35:43,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:43,147 INFO [master/5b617c7edc8e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-27T11:35:43,147 INFO [master/5b617c7edc8e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-27T11:35:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707403152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707403153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,265 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#158 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:43,266 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1c176fb38159417db231f2714f96b723 is 175, key is test_row_0/A:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:43,269 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/ed008587b92246528346656cdc682a9c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/ed008587b92246528346656cdc682a9c 2024-11-27T11:35:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707403262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707403263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,271 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742016_1192 (size=31651) 2024-11-27T11:35:43,276 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into ed008587b92246528346656cdc682a9c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:43,276 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:43,276 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=13, startTime=1732707342826; duration=0sec 2024-11-27T11:35:43,276 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:43,276 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:43,276 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:43,279 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34469 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:43,280 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:43,280 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:43,280 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/586fc0371f21461fbe5cb5e6219d5912, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/09fd59351f6d4cc19aa75d2f1070976f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9609a401f209465cb18d83540cd04edd] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=33.7 K 2024-11-27T11:35:43,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:43,280 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 586fc0371f21461fbe5cb5e6219d5912, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732707339310 2024-11-27T11:35:43,281 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 09fd59351f6d4cc19aa75d2f1070976f, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1732707340460 2024-11-27T11:35:43,284 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112767b127815f3e448eb68bdc65ff01dfb4_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112767b127815f3e448eb68bdc65ff01dfb4_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:43,284 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9609a401f209465cb18d83540cd04edd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732707341155 2024-11-27T11:35:43,285 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1c176fb38159417db231f2714f96b723 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c176fb38159417db231f2714f96b723 2024-11-27T11:35:43,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/45fe1f3f44cd4a4f96842ee052f2a841, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:43,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/45fe1f3f44cd4a4f96842ee052f2a841 is 175, key is test_row_0/A:col10/1732707341937/Put/seqid=0 2024-11-27T11:35:43,295 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into 1c176fb38159417db231f2714f96b723(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:43,295 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:43,295 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=13, startTime=1732707342826; duration=0sec 2024-11-27T11:35:43,295 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:43,295 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:43,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742017_1193 (size=31105) 2024-11-27T11:35:43,314 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#160 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:43,315 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/e12924e08078483b95063dc82552979b is 50, key is test_row_0/C:col10/1732707341155/Put/seqid=0 2024-11-27T11:35:43,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742018_1194 (size=12663) 2024-11-27T11:35:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T11:35:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707403323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707403323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,327 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:43,327 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707403332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,335 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707403482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707403483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,486 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,702 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/45fe1f3f44cd4a4f96842ee052f2a841 2024-11-27T11:35:43,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/925da2fafe284d0f9922ce680ff32cbe is 50, key is test_row_0/B:col10/1732707341937/Put/seqid=0 2024-11-27T11:35:43,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742019_1195 (size=12151) 2024-11-27T11:35:43,727 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/e12924e08078483b95063dc82552979b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/e12924e08078483b95063dc82552979b 2024-11-27T11:35:43,733 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into e12924e08078483b95063dc82552979b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:43,733 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:43,733 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=13, startTime=1732707342826; duration=0sec 2024-11-27T11:35:43,733 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:43,733 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:43,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707403792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707403792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:43,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T11:35:44,116 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/925da2fafe284d0f9922ce680ff32cbe 2024-11-27T11:35:44,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/2ca847c2ef8946e1992884de25aef078 is 50, key is test_row_0/C:col10/1732707341937/Put/seqid=0 2024-11-27T11:35:44,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742020_1196 (size=12151) 2024-11-27T11:35:44,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707404303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:44,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707404303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:44,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:44,529 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/2ca847c2ef8946e1992884de25aef078 2024-11-27T11:35:44,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/45fe1f3f44cd4a4f96842ee052f2a841 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841 2024-11-27T11:35:44,539 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841, entries=150, sequenceid=244, filesize=30.4 K 2024-11-27T11:35:44,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/925da2fafe284d0f9922ce680ff32cbe as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/925da2fafe284d0f9922ce680ff32cbe 2024-11-27T11:35:44,543 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/925da2fafe284d0f9922ce680ff32cbe, entries=150, sequenceid=244, filesize=11.9 K 2024-11-27T11:35:44,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/2ca847c2ef8946e1992884de25aef078 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2ca847c2ef8946e1992884de25aef078 2024-11-27T11:35:44,549 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2ca847c2ef8946e1992884de25aef078, entries=150, sequenceid=244, filesize=11.9 K 2024-11-27T11:35:44,550 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=181.14 KB/185490 for bfc88b634f4291f702daaa0b78cc4f59 in 1701ms, sequenceid=244, compaction requested=false 2024-11-27T11:35:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:44,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-27T11:35:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-27T11:35:44,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-27T11:35:44,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-27T11:35:44,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8610 sec 2024-11-27T11:35:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-27T11:35:44,843 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:44,844 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:44,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-27T11:35:44,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T11:35:44,845 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:44,846 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:44,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:44,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T11:35:44,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-27T11:35:44,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:44,999 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=181.14 KB heapSize=475.36 KB 2024-11-27T11:35:45,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f788ef6ac7b54f50ad9110e05d4a0c3f_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:45,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742021_1197 (size=12454) 2024-11-27T11:35:45,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T11:35:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:45,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707405329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707405330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707405332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707405332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,333 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:45,334 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,334 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707405362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,366 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:45,417 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f788ef6ac7b54f50ad9110e05d4a0c3f_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f788ef6ac7b54f50ad9110e05d4a0c3f_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/d770a662e279495cac4627ab787410ac, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:45,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/d770a662e279495cac4627ab787410ac is 175, key is test_row_0/A:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:45,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742022_1198 (size=31255) 2024-11-27T11:35:45,423 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=60.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/d770a662e279495cac4627ab787410ac 2024-11-27T11:35:45,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/bb201b0ba0974e039ac36fb6201fda6a is 50, key is test_row_0/B:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:45,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742023_1199 (size=12301) 2024-11-27T11:35:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707405443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707405444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T11:35:45,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707405652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707405652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,843 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/bb201b0ba0974e039ac36fb6201fda6a 2024-11-27T11:35:45,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/056e1a68ab644089b341df5c8257bb83 is 50, key is test_row_0/C:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:45,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742024_1200 (size=12301) 2024-11-27T11:35:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707405962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:45,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707405963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:45,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T11:35:46,258 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/056e1a68ab644089b341df5c8257bb83 2024-11-27T11:35:46,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/d770a662e279495cac4627ab787410ac as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac 2024-11-27T11:35:46,268 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac, entries=150, sequenceid=277, filesize=30.5 K 2024-11-27T11:35:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/bb201b0ba0974e039ac36fb6201fda6a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb201b0ba0974e039ac36fb6201fda6a 2024-11-27T11:35:46,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,274 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb201b0ba0974e039ac36fb6201fda6a, entries=150, sequenceid=277, filesize=12.0 K 2024-11-27T11:35:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/056e1a68ab644089b341df5c8257bb83 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/056e1a68ab644089b341df5c8257bb83 2024-11-27T11:35:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,281 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/056e1a68ab644089b341df5c8257bb83, entries=150, sequenceid=277, filesize=12.0 K 2024-11-27T11:35:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,282 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=26.84 KB/27480 for bfc88b634f4291f702daaa0b78cc4f59 in 1283ms, sequenceid=277, compaction requested=true 2024-11-27T11:35:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-27T11:35:46,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-27T11:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-27T11:35:46,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4380 sec 2024-11-27T11:35:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4430 sec 2024-11-27T11:35:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:46,535 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:46,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:35:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,543 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787b49b0ce89446198dfd1c0e558f8c2c_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_1/A:col10/1732707346507/Put/seqid=0 2024-11-27T11:35:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742025_1201 (size=17534) 2024-11-27T11:35:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,552 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,557 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787b49b0ce89446198dfd1c0e558f8c2c_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787b49b0ce89446198dfd1c0e558f8c2c_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,559 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/7caef9a8562c49e38bb9b26f2d42cb8a, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/7caef9a8562c49e38bb9b26f2d42cb8a is 175, key is test_row_1/A:col10/1732707346507/Put/seqid=0 2024-11-27T11:35:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:46,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742026_1202 (size=48635) 2024-11-27T11:35:46,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707406610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707406614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707406722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:46,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:46,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707406722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:46,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,724 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:46,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707406933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:46,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707406933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:46,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:46,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:46,982 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/7caef9a8562c49e38bb9b26f2d42cb8a 2024-11-27T11:35:46,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-27T11:35:46,993 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:46,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/156a2e16248949a19e508d77a17a1ff8 is 50, key is test_row_1/B:col10/1732707346507/Put/seqid=0 2024-11-27T11:35:46,994 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:46,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-27T11:35:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T11:35:46,996 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:46,997 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:46,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:47,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742027_1203 (size=9857) 2024-11-27T11:35:47,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/156a2e16248949a19e508d77a17a1ff8 2024-11-27T11:35:47,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/4b06526ed1bc484da20b345d14ef9c6a is 50, key is test_row_1/C:col10/1732707346507/Put/seqid=0 2024-11-27T11:35:47,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742028_1204 (size=9857) 2024-11-27T11:35:47,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T11:35:47,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T11:35:47,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:47,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:47,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:47,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707407242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:47,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707407242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,246 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:47,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T11:35:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:47,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:47,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:47,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T11:35:47,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/4b06526ed1bc484da20b345d14ef9c6a 2024-11-27T11:35:47,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/7caef9a8562c49e38bb9b26f2d42cb8a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a 2024-11-27T11:35:47,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a, entries=250, sequenceid=288, filesize=47.5 K 2024-11-27T11:35:47,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/156a2e16248949a19e508d77a17a1ff8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/156a2e16248949a19e508d77a17a1ff8 2024-11-27T11:35:47,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/156a2e16248949a19e508d77a17a1ff8, entries=100, sequenceid=288, filesize=9.6 K 2024-11-27T11:35:47,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/4b06526ed1bc484da20b345d14ef9c6a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4b06526ed1bc484da20b345d14ef9c6a 2024-11-27T11:35:47,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4b06526ed1bc484da20b345d14ef9c6a, entries=100, sequenceid=288, filesize=9.6 K 2024-11-27T11:35:47,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for bfc88b634f4291f702daaa0b78cc4f59 in 911ms, sequenceid=288, compaction requested=true 2024-11-27T11:35:47,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:47,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:47,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:47,447 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:47,447 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:47,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:47,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:47,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:47,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:47,449 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142646 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:47,449 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47006 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:47,450 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:47,450 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:47,450 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,450 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,450 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c176fb38159417db231f2714f96b723, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=139.3 K 2024-11-27T11:35:47,450 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/ed008587b92246528346656cdc682a9c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/925da2fafe284d0f9922ce680ff32cbe, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb201b0ba0974e039ac36fb6201fda6a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/156a2e16248949a19e508d77a17a1ff8] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=45.9 K 2024-11-27T11:35:47,450 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,450 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c176fb38159417db231f2714f96b723, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a] 2024-11-27T11:35:47,450 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c176fb38159417db231f2714f96b723, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732707341155 2024-11-27T11:35:47,450 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ed008587b92246528346656cdc682a9c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732707341155 2024-11-27T11:35:47,452 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45fe1f3f44cd4a4f96842ee052f2a841, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732707341936 2024-11-27T11:35:47,452 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 925da2fafe284d0f9922ce680ff32cbe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732707341936 2024-11-27T11:35:47,452 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d770a662e279495cac4627ab787410ac, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732707343138 2024-11-27T11:35:47,452 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting bb201b0ba0974e039ac36fb6201fda6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732707343138 2024-11-27T11:35:47,452 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7caef9a8562c49e38bb9b26f2d42cb8a, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732707345328 2024-11-27T11:35:47,453 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 156a2e16248949a19e508d77a17a1ff8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732707346507 2024-11-27T11:35:47,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-27T11:35:47,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,457 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:35:47,463 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:47,465 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127148dda46773941918c2d97f699f5d6a5_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:47,467 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127148dda46773941918c2d97f699f5d6a5_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:47,468 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127148dda46773941918c2d97f699f5d6a5_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0d9bba24d0b4f59b2d96289c0d9b7a7_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707346602/Put/seqid=0 2024-11-27T11:35:47,475 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#171 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:47,476 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/e58d2b8df1ca4612a558fe7a973f8b14 is 50, key is test_row_0/B:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:47,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742029_1205 (size=4469) 2024-11-27T11:35:47,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742030_1206 (size=12454) 2024-11-27T11:35:47,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742031_1207 (size=12983) 2024-11-27T11:35:47,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T11:35:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:47,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:47,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:47,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707407768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:47,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:47,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707407771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,772 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:47,883 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#169 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:47,884 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/8c36f5a826de4941bd3e727f9918bb45 is 175, key is test_row_0/A:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:47,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742032_1208 (size=32044) 2024-11-27T11:35:47,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:47,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707407882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:47,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:47,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707407882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:47,895 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d0d9bba24d0b4f59b2d96289c0d9b7a7_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0d9bba24d0b4f59b2d96289c0d9b7a7_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:47,896 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/e58d2b8df1ca4612a558fe7a973f8b14 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/e58d2b8df1ca4612a558fe7a973f8b14 2024-11-27T11:35:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6a90f5cd48654f5d889df75f028cf1b5, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:47,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6a90f5cd48654f5d889df75f028cf1b5 is 175, key is test_row_0/A:col10/1732707346602/Put/seqid=0 2024-11-27T11:35:47,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742033_1209 (size=31255) 2024-11-27T11:35:47,903 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into e58d2b8df1ca4612a558fe7a973f8b14(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:47,903 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:47,903 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=12, startTime=1732707347447; duration=0sec 2024-11-27T11:35:47,903 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:47,903 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:47,903 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:35:47,904 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46972 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:35:47,904 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:47,904 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:47,905 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/e12924e08078483b95063dc82552979b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2ca847c2ef8946e1992884de25aef078, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/056e1a68ab644089b341df5c8257bb83, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4b06526ed1bc484da20b345d14ef9c6a] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=45.9 K 2024-11-27T11:35:47,905 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e12924e08078483b95063dc82552979b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732707341155 2024-11-27T11:35:47,906 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca847c2ef8946e1992884de25aef078, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1732707341936 2024-11-27T11:35:47,906 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 056e1a68ab644089b341df5c8257bb83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1732707343138 2024-11-27T11:35:47,906 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b06526ed1bc484da20b345d14ef9c6a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732707346507 2024-11-27T11:35:47,918 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:47,919 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/f67f2bad747d4832888d36f37a3ed5aa is 50, key is test_row_0/C:col10/1732707343151/Put/seqid=0 2024-11-27T11:35:47,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742034_1210 (size=12949) 2024-11-27T11:35:48,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707408102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:48,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:48,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707408102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T11:35:48,293 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/8c36f5a826de4941bd3e727f9918bb45 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/8c36f5a826de4941bd3e727f9918bb45 2024-11-27T11:35:48,313 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6a90f5cd48654f5d889df75f028cf1b5 2024-11-27T11:35:48,314 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into 8c36f5a826de4941bd3e727f9918bb45(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:48,315 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:48,315 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=12, startTime=1732707347446; duration=0sec 2024-11-27T11:35:48,315 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:48,315 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:48,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d369b43f563949709d0a735d11eb2bf4 is 50, key is test_row_0/B:col10/1732707346602/Put/seqid=0 2024-11-27T11:35:48,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742035_1211 (size=12301) 2024-11-27T11:35:48,329 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d369b43f563949709d0a735d11eb2bf4 2024-11-27T11:35:48,330 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/f67f2bad747d4832888d36f37a3ed5aa as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f67f2bad747d4832888d36f37a3ed5aa 2024-11-27T11:35:48,336 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into f67f2bad747d4832888d36f37a3ed5aa(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:48,336 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:48,336 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=12, startTime=1732707347447; duration=0sec 2024-11-27T11:35:48,336 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:48,336 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:48,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/0219a63a27cf4fc48a1edced0cbb5640 is 50, key is test_row_0/C:col10/1732707346602/Put/seqid=0 2024-11-27T11:35:48,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742036_1212 (size=12301) 2024-11-27T11:35:48,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:48,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707408413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:48,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707408413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:48,745 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/0219a63a27cf4fc48a1edced0cbb5640 2024-11-27T11:35:48,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/6a90f5cd48654f5d889df75f028cf1b5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5 2024-11-27T11:35:48,754 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5, entries=150, sequenceid=313, filesize=30.5 K 2024-11-27T11:35:48,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d369b43f563949709d0a735d11eb2bf4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d369b43f563949709d0a735d11eb2bf4 2024-11-27T11:35:48,759 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d369b43f563949709d0a735d11eb2bf4, entries=150, sequenceid=313, filesize=12.0 K 2024-11-27T11:35:48,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/0219a63a27cf4fc48a1edced0cbb5640 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0219a63a27cf4fc48a1edced0cbb5640 2024-11-27T11:35:48,764 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0219a63a27cf4fc48a1edced0cbb5640, entries=150, sequenceid=313, filesize=12.0 K 2024-11-27T11:35:48,765 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for bfc88b634f4291f702daaa0b78cc4f59 in 1308ms, sequenceid=313, compaction requested=false 2024-11-27T11:35:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:48,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-27T11:35:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-27T11:35:48,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-27T11:35:48,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7700 sec 2024-11-27T11:35:48,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.7750 sec 2024-11-27T11:35:48,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:48,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:48,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:35:48,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279cafd3bc0e0b4233a8fba63e12e01da5_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:48,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742037_1213 (size=17534) 2024-11-27T11:35:48,938 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:48,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279cafd3bc0e0b4233a8fba63e12e01da5_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279cafd3bc0e0b4233a8fba63e12e01da5_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:48,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/43d41f8cd3334e10987edcda1a95b179, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:48,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/43d41f8cd3334e10987edcda1a95b179 is 175, key is test_row_0/A:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:48,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742038_1214 (size=48639) 2024-11-27T11:35:48,949 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/43d41f8cd3334e10987edcda1a95b179 2024-11-27T11:35:48,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/bb5cbbad8cd14c0b9b20de06e24c4aa0 is 50, key is test_row_0/B:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:48,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742039_1215 (size=12301) 2024-11-27T11:35:48,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:48,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707408980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:48,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:48,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707408983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:48,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:48,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707409093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,097 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707409102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-27T11:35:49,143 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:49,143 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:35:49,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-27T11:35:49,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T11:35:49,145 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:35:49,146 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:35:49,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:35:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T11:35:49,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T11:35:49,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:49,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707409303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707409313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,316 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57802 deadline: 1732707409352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57768 deadline: 1732707409352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,355 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:49,355 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/bb5cbbad8cd14c0b9b20de06e24c4aa0 2024-11-27T11:35:49,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/ab5e6d2f9f774c5d8146fec8323dde12 is 50, key is test_row_0/C:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:49,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742040_1216 (size=12301) 2024-11-27T11:35:49,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57792 deadline: 1732707409383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,384 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in bfc88b634f4291f702daaa0b78cc4f59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:35:49,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T11:35:49,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:49,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T11:35:49,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T11:35:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707409613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,618 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707409622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:49,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:49,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T11:35:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:35:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T11:35:49,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/ab5e6d2f9f774c5d8146fec8323dde12 2024-11-27T11:35:49,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/43d41f8cd3334e10987edcda1a95b179 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179 2024-11-27T11:35:49,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179, entries=250, sequenceid=329, filesize=47.5 K 2024-11-27T11:35:49,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/bb5cbbad8cd14c0b9b20de06e24c4aa0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb5cbbad8cd14c0b9b20de06e24c4aa0 2024-11-27T11:35:49,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb5cbbad8cd14c0b9b20de06e24c4aa0, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T11:35:49,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/ab5e6d2f9f774c5d8146fec8323dde12 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/ab5e6d2f9f774c5d8146fec8323dde12 2024-11-27T11:35:49,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/ab5e6d2f9f774c5d8146fec8323dde12, entries=150, sequenceid=329, filesize=12.0 K 2024-11-27T11:35:49,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for bfc88b634f4291f702daaa0b78cc4f59 in 877ms, sequenceid=329, compaction requested=true 2024-11-27T11:35:49,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:49,802 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:49,802 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:49,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:49,803 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:49,803 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111938 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:49,803 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:49,803 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:49,803 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,803 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,803 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/e58d2b8df1ca4612a558fe7a973f8b14, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d369b43f563949709d0a735d11eb2bf4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb5cbbad8cd14c0b9b20de06e24c4aa0] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=36.7 K 2024-11-27T11:35:49,803 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/8c36f5a826de4941bd3e727f9918bb45, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=109.3 K 2024-11-27T11:35:49,803 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,803 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/8c36f5a826de4941bd3e727f9918bb45, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179] 2024-11-27T11:35:49,804 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e58d2b8df1ca4612a558fe7a973f8b14, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732707343151 2024-11-27T11:35:49,804 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d369b43f563949709d0a735d11eb2bf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732707346602 2024-11-27T11:35:49,804 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c36f5a826de4941bd3e727f9918bb45, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732707343151 2024-11-27T11:35:49,804 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting bb5cbbad8cd14c0b9b20de06e24c4aa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732707347762 2024-11-27T11:35:49,804 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a90f5cd48654f5d889df75f028cf1b5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732707346602 2024-11-27T11:35:49,805 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43d41f8cd3334e10987edcda1a95b179, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732707347753 2024-11-27T11:35:49,812 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:49,813 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#178 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:49,813 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/72abf78939ac4701a86649277725dd31 is 50, key is test_row_0/B:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:49,816 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411276f2c3f483a8842a2a3fbb62c9b536f92_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:49,818 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411276f2c3f483a8842a2a3fbb62c9b536f92_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:49,819 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411276f2c3f483a8842a2a3fbb62c9b536f92_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:49,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742041_1217 (size=13085) 2024-11-27T11:35:49,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742042_1218 (size=4469) 2024-11-27T11:35:49,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-27T11:35:49,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:49,911 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:35:49,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127704932bbaf3a4bd4856be8e7776d0c06_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707348982/Put/seqid=0 2024-11-27T11:35:49,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742043_1219 (size=12454) 2024-11-27T11:35:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:50,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. as already flushing 2024-11-27T11:35:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707410147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707410156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,227 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#179 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:50,228 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cdd7f089afe34ab385efd75eb1028e3e is 175, key is test_row_0/A:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:50,228 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/72abf78939ac4701a86649277725dd31 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/72abf78939ac4701a86649277725dd31 2024-11-27T11:35:50,234 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into 72abf78939ac4701a86649277725dd31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:50,234 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:50,234 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=13, startTime=1732707349802; duration=0sec 2024-11-27T11:35:50,234 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:50,234 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:50,234 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:50,235 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:50,235 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:50,236 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:50,236 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f67f2bad747d4832888d36f37a3ed5aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0219a63a27cf4fc48a1edced0cbb5640, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/ab5e6d2f9f774c5d8146fec8323dde12] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=36.7 K 2024-11-27T11:35:50,236 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f67f2bad747d4832888d36f37a3ed5aa, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732707343151 2024-11-27T11:35:50,237 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0219a63a27cf4fc48a1edced0cbb5640, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1732707346602 2024-11-27T11:35:50,237 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5e6d2f9f774c5d8146fec8323dde12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732707347762 2024-11-27T11:35:50,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742044_1220 (size=32039) 2024-11-27T11:35:50,246 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#181 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:50,247 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/d9b4568b52d246feb89997b447b3be09 is 50, key is test_row_0/C:col10/1732707347762/Put/seqid=0 2024-11-27T11:35:50,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742045_1221 (size=13051) 2024-11-27T11:35:50,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707410262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707410262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T11:35:50,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:50,330 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127704932bbaf3a4bd4856be8e7776d0c06_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127704932bbaf3a4bd4856be8e7776d0c06_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/f728536e6d7c4c1c840a463b87589014, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:50,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/f728536e6d7c4c1c840a463b87589014 is 175, key is test_row_0/A:col10/1732707348982/Put/seqid=0 2024-11-27T11:35:50,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742046_1222 (size=31255) 2024-11-27T11:35:50,335 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=352, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/f728536e6d7c4c1c840a463b87589014 2024-11-27T11:35:50,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/6e80703966db4de2b0213aa4f2d2ee73 is 50, key is test_row_0/B:col10/1732707348982/Put/seqid=0 2024-11-27T11:35:50,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742047_1223 (size=12301) 2024-11-27T11:35:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707410473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707410473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,643 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/cdd7f089afe34ab385efd75eb1028e3e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cdd7f089afe34ab385efd75eb1028e3e 2024-11-27T11:35:50,648 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into cdd7f089afe34ab385efd75eb1028e3e(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:50,648 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:50,648 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=13, startTime=1732707349802; duration=0sec 2024-11-27T11:35:50,648 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:50,648 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:50,664 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/d9b4568b52d246feb89997b447b3be09 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/d9b4568b52d246feb89997b447b3be09 2024-11-27T11:35:50,670 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into d9b4568b52d246feb89997b447b3be09(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:50,670 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:50,670 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=13, startTime=1732707349802; duration=0sec 2024-11-27T11:35:50,670 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:50,670 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:50,747 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/6e80703966db4de2b0213aa4f2d2ee73 2024-11-27T11:35:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/887fe7165da54d268678e1b8b4de31b1 is 50, key is test_row_0/C:col10/1732707348982/Put/seqid=0 2024-11-27T11:35:50,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742048_1224 (size=12301) 2024-11-27T11:35:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57722 deadline: 1732707410782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:35:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57800 deadline: 1732707410782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:35:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:35:50,932 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-27T11:35:50,932 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:50,932 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,932 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,932 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:50,936 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-27T11:35:50,936 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:50,936 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,936 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,936 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:50,940 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-27T11:35:50,940 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:50,940 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,940 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:50,942 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-27T11:35:50,942 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:50,942 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,942 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:50,942 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:51,159 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/887fe7165da54d268678e1b8b4de31b1 2024-11-27T11:35:51,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/f728536e6d7c4c1c840a463b87589014 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014 2024-11-27T11:35:51,168 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014, entries=150, sequenceid=352, filesize=30.5 K 2024-11-27T11:35:51,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/6e80703966db4de2b0213aa4f2d2ee73 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6e80703966db4de2b0213aa4f2d2ee73 2024-11-27T11:35:51,172 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6e80703966db4de2b0213aa4f2d2ee73, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T11:35:51,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/887fe7165da54d268678e1b8b4de31b1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/887fe7165da54d268678e1b8b4de31b1 2024-11-27T11:35:51,176 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/887fe7165da54d268678e1b8b4de31b1, entries=150, sequenceid=352, filesize=12.0 K 2024-11-27T11:35:51,177 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for bfc88b634f4291f702daaa0b78cc4f59 in 1266ms, sequenceid=352, compaction requested=false 2024-11-27T11:35:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:51,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-27T11:35:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-27T11:35:51,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-27T11:35:51,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0320 sec 2024-11-27T11:35:51,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.0370 sec 2024-11-27T11:35:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-27T11:35:51,293 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:35:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:51,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:35:51,294 INFO [Thread-634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-634. 2024-11-27T11:35:51,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T11:35:51,294 INFO [Thread-636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-636. 2024-11-27T11:35:51,294 DEBUG [Thread-634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:51,294 DEBUG [Thread-636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:51,294 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:51,294 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:51,295 DEBUG [Thread-634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:51,295 DEBUG [Thread-636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:51,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:51,295 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:51,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112726c2c00bbe2744c0b2b2d4794b60eff2_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_0/A:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:51,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742049_1225 (size=12454) 2024-11-27T11:35:51,705 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:35:51,709 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112726c2c00bbe2744c0b2b2d4794b60eff2_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112726c2c00bbe2744c0b2b2d4794b60eff2_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:51,709 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9a9d5c4a8a9744d1ac15743bbc55f9b1, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:51,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9a9d5c4a8a9744d1ac15743bbc55f9b1 is 175, key is test_row_0/A:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:51,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742050_1226 (size=31255) 2024-11-27T11:35:52,114 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9a9d5c4a8a9744d1ac15743bbc55f9b1 2024-11-27T11:35:52,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/88bb4905a926424bab370b814477031f is 50, key is test_row_0/B:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:52,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742051_1227 (size=12301) 2024-11-27T11:35:52,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/88bb4905a926424bab370b814477031f 2024-11-27T11:35:52,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/470a1a8c97a6444990aaafab375d58b1 is 50, key is test_row_0/C:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:52,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742052_1228 (size=12301) 2024-11-27T11:35:52,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/470a1a8c97a6444990aaafab375d58b1 2024-11-27T11:35:52,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/9a9d5c4a8a9744d1ac15743bbc55f9b1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1 2024-11-27T11:35:52,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1, entries=150, sequenceid=370, filesize=30.5 K 2024-11-27T11:35:52,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/88bb4905a926424bab370b814477031f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/88bb4905a926424bab370b814477031f 2024-11-27T11:35:52,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/88bb4905a926424bab370b814477031f, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T11:35:52,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/470a1a8c97a6444990aaafab375d58b1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/470a1a8c97a6444990aaafab375d58b1 2024-11-27T11:35:52,952 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/470a1a8c97a6444990aaafab375d58b1, entries=150, sequenceid=370, filesize=12.0 K 2024-11-27T11:35:52,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for bfc88b634f4291f702daaa0b78cc4f59 in 1659ms, sequenceid=370, compaction requested=true 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bfc88b634f4291f702daaa0b78cc4f59:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:35:52,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:52,953 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:52,953 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:52,954 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:52,954 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:52,954 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/B is initiating minor compaction (all files) 2024-11-27T11:35:52,954 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/A is initiating minor compaction (all files) 2024-11-27T11:35:52,954 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/A in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:52,954 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/B in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:52,954 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cdd7f089afe34ab385efd75eb1028e3e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=92.3 K 2024-11-27T11:35:52,954 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/72abf78939ac4701a86649277725dd31, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6e80703966db4de2b0213aa4f2d2ee73, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/88bb4905a926424bab370b814477031f] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=36.8 K 2024-11-27T11:35:52,954 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:52,954 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cdd7f089afe34ab385efd75eb1028e3e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1] 2024-11-27T11:35:52,955 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 72abf78939ac4701a86649277725dd31, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732707347762 2024-11-27T11:35:52,955 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd7f089afe34ab385efd75eb1028e3e, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732707347762 2024-11-27T11:35:52,955 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e80703966db4de2b0213aa4f2d2ee73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732707348966 2024-11-27T11:35:52,955 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f728536e6d7c4c1c840a463b87589014, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732707348966 2024-11-27T11:35:52,955 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bb4905a926424bab370b814477031f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732707350146 2024-11-27T11:35:52,955 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a9d5c4a8a9744d1ac15743bbc55f9b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732707350146 2024-11-27T11:35:52,962 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:52,963 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#B#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:52,964 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127b2f2657542e44e3fa6bce0f7e7ff5a42_bfc88b634f4291f702daaa0b78cc4f59 store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:52,964 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d6574dd73a3b420abc5f60e4c8a005c8 is 50, key is test_row_0/B:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:52,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742053_1229 (size=13187) 2024-11-27T11:35:52,991 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127b2f2657542e44e3fa6bce0f7e7ff5a42_bfc88b634f4291f702daaa0b78cc4f59, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:52,991 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127b2f2657542e44e3fa6bce0f7e7ff5a42_bfc88b634f4291f702daaa0b78cc4f59 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:35:52,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742054_1230 (size=4469) 2024-11-27T11:35:53,373 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d6574dd73a3b420abc5f60e4c8a005c8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d6574dd73a3b420abc5f60e4c8a005c8 2024-11-27T11:35:53,378 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/B of bfc88b634f4291f702daaa0b78cc4f59 into d6574dd73a3b420abc5f60e4c8a005c8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:53,378 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:53,378 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/B, priority=13, startTime=1732707352953; duration=0sec 2024-11-27T11:35:53,378 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:35:53,378 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:B 2024-11-27T11:35:53,378 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:35:53,379 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:35:53,379 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): bfc88b634f4291f702daaa0b78cc4f59/C is initiating minor compaction (all files) 2024-11-27T11:35:53,379 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bfc88b634f4291f702daaa0b78cc4f59/C in TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:53,379 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/d9b4568b52d246feb89997b447b3be09, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/887fe7165da54d268678e1b8b4de31b1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/470a1a8c97a6444990aaafab375d58b1] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp, totalSize=36.8 K 2024-11-27T11:35:53,380 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b4568b52d246feb89997b447b3be09, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732707347762 2024-11-27T11:35:53,380 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 887fe7165da54d268678e1b8b4de31b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1732707348966 2024-11-27T11:35:53,380 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 470a1a8c97a6444990aaafab375d58b1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=370, earliestPutTs=1732707350146 2024-11-27T11:35:53,388 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#C#compaction#189 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:53,389 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/bcbf226ebe96448fbe734926a40250b0 is 50, key is test_row_0/C:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:53,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742055_1231 (size=13153) 2024-11-27T11:35:53,395 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bfc88b634f4291f702daaa0b78cc4f59#A#compaction#188 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:35:53,396 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/2f9615459fbc42bb99a6fec00baa971a is 175, key is test_row_0/A:col10/1732707351293/Put/seqid=0 2024-11-27T11:35:53,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742056_1232 (size=32141) 2024-11-27T11:35:53,797 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/bcbf226ebe96448fbe734926a40250b0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/bcbf226ebe96448fbe734926a40250b0 2024-11-27T11:35:53,802 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/C of bfc88b634f4291f702daaa0b78cc4f59 into bcbf226ebe96448fbe734926a40250b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:53,802 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:53,802 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/C, priority=13, startTime=1732707352953; duration=0sec 2024-11-27T11:35:53,802 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:53,802 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:C 2024-11-27T11:35:53,803 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/2f9615459fbc42bb99a6fec00baa971a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/2f9615459fbc42bb99a6fec00baa971a 2024-11-27T11:35:53,807 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bfc88b634f4291f702daaa0b78cc4f59/A of bfc88b634f4291f702daaa0b78cc4f59 into 2f9615459fbc42bb99a6fec00baa971a(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:35:53,807 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bfc88b634f4291f702daaa0b78cc4f59: 2024-11-27T11:35:53,807 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., storeName=bfc88b634f4291f702daaa0b78cc4f59/A, priority=13, startTime=1732707352953; duration=0sec 2024-11-27T11:35:53,807 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:35:53,807 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bfc88b634f4291f702daaa0b78cc4f59:A 2024-11-27T11:35:57,432 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T11:35:59,374 INFO [Thread-630 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-630. 2024-11-27T11:35:59,374 DEBUG [Thread-630 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:59,374 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,374 DEBUG [Thread-630 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,374 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:59,383 INFO [Thread-632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-632. 2024-11-27T11:35:59,383 DEBUG [Thread-632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:59,383 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,384 DEBUG [Thread-632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,384 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:59,424 INFO [Thread-638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-638. 2024-11-27T11:35:59,424 DEBUG [Thread-638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:35:59,424 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,424 DEBUG [Thread-638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,424 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 24 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 109 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3582 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3628 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1516 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4545 rows 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1519 2024-11-27T11:35:59,424 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4557 rows 2024-11-27T11:35:59,424 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:35:59,425 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:35:59,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:35:59,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:35:59,425 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T11:35:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T11:35:59,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T11:35:59,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707359428"}]},"ts":"1732707359428"} 2024-11-27T11:35:59,429 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T11:35:59,429 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T11:35:59,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:35:59,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, UNASSIGN}] 2024-11-27T11:35:59,432 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, UNASSIGN 2024-11-27T11:35:59,433 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:35:59,434 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, UNASSIGN because future has completed 2024-11-27T11:35:59,434 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:35:59,434 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:35:59,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T11:35:59,586 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:35:59,586 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:35:59,586 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing bfc88b634f4291f702daaa0b78cc4f59, disabling compactions & flushes 2024-11-27T11:35:59,586 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:59,586 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:59,586 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. after waiting 0 ms 2024-11-27T11:35:59,586 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:35:59,587 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing bfc88b634f4291f702daaa0b78cc4f59 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-27T11:35:59,593 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271df0f162c93941c38ab6760869ab9eb2_bfc88b634f4291f702daaa0b78cc4f59 is 50, key is test_row_1/A:col10/1732707359423/Put/seqid=0 2024-11-27T11:35:59,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742057_1233 (size=9914) 2024-11-27T11:35:59,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T11:35:59,997 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:00,001 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271df0f162c93941c38ab6760869ab9eb2_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271df0f162c93941c38ab6760869ab9eb2_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:00,002 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1c873914e79d431db5149b12b32ef2ba, store: [table=TestAcidGuarantees family=A region=bfc88b634f4291f702daaa0b78cc4f59] 2024-11-27T11:36:00,002 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1c873914e79d431db5149b12b32ef2ba is 175, key is test_row_1/A:col10/1732707359423/Put/seqid=0 2024-11-27T11:36:00,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742058_1234 (size=22561) 2024-11-27T11:36:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T11:36:00,407 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=379, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1c873914e79d431db5149b12b32ef2ba 2024-11-27T11:36:00,413 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d872b0014cff4bb3bb316dae1ade6633 is 50, key is test_row_1/B:col10/1732707359423/Put/seqid=0 2024-11-27T11:36:00,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742059_1235 (size=9857) 2024-11-27T11:36:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T11:36:00,818 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d872b0014cff4bb3bb316dae1ade6633 2024-11-27T11:36:00,825 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/2be48a73765c40ba8d9fc1b55e18c4d5 is 50, key is test_row_1/C:col10/1732707359423/Put/seqid=0 2024-11-27T11:36:00,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742060_1236 (size=9857) 2024-11-27T11:36:01,229 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/2be48a73765c40ba8d9fc1b55e18c4d5 2024-11-27T11:36:01,233 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/A/1c873914e79d431db5149b12b32ef2ba as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c873914e79d431db5149b12b32ef2ba 2024-11-27T11:36:01,237 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c873914e79d431db5149b12b32ef2ba, entries=100, sequenceid=379, filesize=22.0 K 2024-11-27T11:36:01,237 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/B/d872b0014cff4bb3bb316dae1ade6633 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d872b0014cff4bb3bb316dae1ade6633 2024-11-27T11:36:01,241 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d872b0014cff4bb3bb316dae1ade6633, entries=100, sequenceid=379, filesize=9.6 K 2024-11-27T11:36:01,242 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/.tmp/C/2be48a73765c40ba8d9fc1b55e18c4d5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2be48a73765c40ba8d9fc1b55e18c4d5 2024-11-27T11:36:01,245 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2be48a73765c40ba8d9fc1b55e18c4d5, entries=100, sequenceid=379, filesize=9.6 K 2024-11-27T11:36:01,246 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for bfc88b634f4291f702daaa0b78cc4f59 in 1660ms, sequenceid=379, compaction requested=false 2024-11-27T11:36:01,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/abaaace8a0c445fe9be257e37001d1d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/c58a54aaf7f546cf96887d1586facb33, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d5dc9fea5bc245e6bd79b170ac9f0846, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9712c9c17b584f52a813cc41b96c9e64, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c176fb38159417db231f2714f96b723, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/8c36f5a826de4941bd3e727f9918bb45, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cdd7f089afe34ab385efd75eb1028e3e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1] to archive 2024-11-27T11:36:01,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:36:01,249 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6fb0d9df9c094f3ca3b81c05226e7e49 2024-11-27T11:36:01,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cd3845958ffb4b359c6012b3c27b7584 2024-11-27T11:36:01,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/864a04047c1c41a1b00417d753c978ab 2024-11-27T11:36:01,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/abaaace8a0c445fe9be257e37001d1d6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/abaaace8a0c445fe9be257e37001d1d6 2024-11-27T11:36:01,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e6242b8d6f2d4aea9444ed713fe79e89 2024-11-27T11:36:01,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cbef93e810a346ffbc5a1240f96da2a0 2024-11-27T11:36:01,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/c58a54aaf7f546cf96887d1586facb33 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/c58a54aaf7f546cf96887d1586facb33 2024-11-27T11:36:01,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9c8fdd35ce5d4da386677fefdcb91d55 2024-11-27T11:36:01,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/e81723df27f3462b83fe6fa73a25b607 2024-11-27T11:36:01,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d5dc9fea5bc245e6bd79b170ac9f0846 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d5dc9fea5bc245e6bd79b170ac9f0846 2024-11-27T11:36:01,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/aba95792b12744549c72f1251840fe35 2024-11-27T11:36:01,259 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/529c1bc1591d4ddea34be71d09f7d1aa 2024-11-27T11:36:01,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/3f1b2c137aaf4dd180663675aff6f112 2024-11-27T11:36:01,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9712c9c17b584f52a813cc41b96c9e64 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9712c9c17b584f52a813cc41b96c9e64 2024-11-27T11:36:01,262 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1fcc5ce7ad2b44c0bd746633c3af8c07 2024-11-27T11:36:01,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/71cb98c703d04a4e841c5bd23c68b81e 2024-11-27T11:36:01,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c176fb38159417db231f2714f96b723 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c176fb38159417db231f2714f96b723 2024-11-27T11:36:01,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/45fe1f3f44cd4a4f96842ee052f2a841 2024-11-27T11:36:01,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/d770a662e279495cac4627ab787410ac 2024-11-27T11:36:01,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/7caef9a8562c49e38bb9b26f2d42cb8a 2024-11-27T11:36:01,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/8c36f5a826de4941bd3e727f9918bb45 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/8c36f5a826de4941bd3e727f9918bb45 2024-11-27T11:36:01,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/6a90f5cd48654f5d889df75f028cf1b5 2024-11-27T11:36:01,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/43d41f8cd3334e10987edcda1a95b179 2024-11-27T11:36:01,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cdd7f089afe34ab385efd75eb1028e3e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/cdd7f089afe34ab385efd75eb1028e3e 2024-11-27T11:36:01,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/f728536e6d7c4c1c840a463b87589014 2024-11-27T11:36:01,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/9a9d5c4a8a9744d1ac15743bbc55f9b1 2024-11-27T11:36:01,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/177f9d73e283443387c4927ba4247e2f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/5086b025b9df4d8189612359db78b215, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be6fe9c09fbe4aa2be43d292ed94febc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/b30f9fe6ad284563a92d391bf5d48e54, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/411ab17cbe464f4f8d059b148321017e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8cd8005730bb4eadb0519bbe58fc0099, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/3bf749f857894e92b4689f91dc087964, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/f6ae478963954a8b84875f6347e50ca2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6f93082e28694724969f2fc5a0f8507e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/54022562d96a4274bf7e9cd4ecdda4d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/37a540b175a442b4b8238008a8c2beb7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be93484bdb724360837efe30e1867e83, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/7caf977aa7294ffaad210a7d32cd5204, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d1326a2e4c544653a4667daebe64bf1c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/96f885ea1dc342b0aca87315b1bc1b6d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/ed008587b92246528346656cdc682a9c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8102d3db7a83432a913302af33b6363a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/925da2fafe284d0f9922ce680ff32cbe, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb201b0ba0974e039ac36fb6201fda6a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/e58d2b8df1ca4612a558fe7a973f8b14, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/156a2e16248949a19e508d77a17a1ff8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d369b43f563949709d0a735d11eb2bf4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/72abf78939ac4701a86649277725dd31, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb5cbbad8cd14c0b9b20de06e24c4aa0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6e80703966db4de2b0213aa4f2d2ee73, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/88bb4905a926424bab370b814477031f] to archive 2024-11-27T11:36:01,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:36:01,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/177f9d73e283443387c4927ba4247e2f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/177f9d73e283443387c4927ba4247e2f 2024-11-27T11:36:01,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/5086b025b9df4d8189612359db78b215 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/5086b025b9df4d8189612359db78b215 2024-11-27T11:36:01,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be6fe9c09fbe4aa2be43d292ed94febc to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be6fe9c09fbe4aa2be43d292ed94febc 2024-11-27T11:36:01,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/b30f9fe6ad284563a92d391bf5d48e54 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/b30f9fe6ad284563a92d391bf5d48e54 2024-11-27T11:36:01,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/411ab17cbe464f4f8d059b148321017e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/411ab17cbe464f4f8d059b148321017e 2024-11-27T11:36:01,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8cd8005730bb4eadb0519bbe58fc0099 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8cd8005730bb4eadb0519bbe58fc0099 2024-11-27T11:36:01,283 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/3bf749f857894e92b4689f91dc087964 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/3bf749f857894e92b4689f91dc087964 2024-11-27T11:36:01,284 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/f6ae478963954a8b84875f6347e50ca2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/f6ae478963954a8b84875f6347e50ca2 2024-11-27T11:36:01,285 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6f93082e28694724969f2fc5a0f8507e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6f93082e28694724969f2fc5a0f8507e 2024-11-27T11:36:01,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/54022562d96a4274bf7e9cd4ecdda4d5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/54022562d96a4274bf7e9cd4ecdda4d5 2024-11-27T11:36:01,286 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/37a540b175a442b4b8238008a8c2beb7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/37a540b175a442b4b8238008a8c2beb7 2024-11-27T11:36:01,287 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be93484bdb724360837efe30e1867e83 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/be93484bdb724360837efe30e1867e83 2024-11-27T11:36:01,288 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/7caf977aa7294ffaad210a7d32cd5204 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/7caf977aa7294ffaad210a7d32cd5204 2024-11-27T11:36:01,289 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d1326a2e4c544653a4667daebe64bf1c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d1326a2e4c544653a4667daebe64bf1c 2024-11-27T11:36:01,290 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/96f885ea1dc342b0aca87315b1bc1b6d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/96f885ea1dc342b0aca87315b1bc1b6d 2024-11-27T11:36:01,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/ed008587b92246528346656cdc682a9c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/ed008587b92246528346656cdc682a9c 2024-11-27T11:36:01,291 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8102d3db7a83432a913302af33b6363a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/8102d3db7a83432a913302af33b6363a 2024-11-27T11:36:01,292 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/925da2fafe284d0f9922ce680ff32cbe to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/925da2fafe284d0f9922ce680ff32cbe 2024-11-27T11:36:01,293 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb201b0ba0974e039ac36fb6201fda6a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb201b0ba0974e039ac36fb6201fda6a 2024-11-27T11:36:01,294 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/e58d2b8df1ca4612a558fe7a973f8b14 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/e58d2b8df1ca4612a558fe7a973f8b14 2024-11-27T11:36:01,295 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/156a2e16248949a19e508d77a17a1ff8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/156a2e16248949a19e508d77a17a1ff8 2024-11-27T11:36:01,296 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d369b43f563949709d0a735d11eb2bf4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d369b43f563949709d0a735d11eb2bf4 2024-11-27T11:36:01,297 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/72abf78939ac4701a86649277725dd31 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/72abf78939ac4701a86649277725dd31 2024-11-27T11:36:01,298 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb5cbbad8cd14c0b9b20de06e24c4aa0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/bb5cbbad8cd14c0b9b20de06e24c4aa0 2024-11-27T11:36:01,299 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6e80703966db4de2b0213aa4f2d2ee73 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/6e80703966db4de2b0213aa4f2d2ee73 2024-11-27T11:36:01,300 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/88bb4905a926424bab370b814477031f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/88bb4905a926424bab370b814477031f 2024-11-27T11:36:01,301 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/748c8f56f9ba45d5b136766bcaaa56d2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/00db576d4f824b5facfd3538442d067a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/20140bbeabf04269b9806cf14283cc76, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/3d8a804094f9456099dc7d2e300d4266, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/88bd743566364b49b412e0a8b7b605b4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4a1dd1f7111f466b858cd98e9e4f6196, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f20c55261a4844249a7c5387658b74d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9522d497dad24e70ac6cbe3bfccd87d1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/19b5e8b9086e4b088646ae0065de8a59, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/db1f81f168594040a345a4c4dd5f1b04, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0b1cd520766b419681b37f351d37df12, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/586fc0371f21461fbe5cb5e6219d5912, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/939f8799b9af4003b19813629d0d3100, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/09fd59351f6d4cc19aa75d2f1070976f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/e12924e08078483b95063dc82552979b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9609a401f209465cb18d83540cd04edd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2ca847c2ef8946e1992884de25aef078, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/056e1a68ab644089b341df5c8257bb83, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f67f2bad747d4832888d36f37a3ed5aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4b06526ed1bc484da20b345d14ef9c6a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0219a63a27cf4fc48a1edced0cbb5640, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/d9b4568b52d246feb89997b447b3be09, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/ab5e6d2f9f774c5d8146fec8323dde12, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/887fe7165da54d268678e1b8b4de31b1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/470a1a8c97a6444990aaafab375d58b1] to archive 2024-11-27T11:36:01,302 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:36:01,304 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/748c8f56f9ba45d5b136766bcaaa56d2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/748c8f56f9ba45d5b136766bcaaa56d2 2024-11-27T11:36:01,305 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/00db576d4f824b5facfd3538442d067a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/00db576d4f824b5facfd3538442d067a 2024-11-27T11:36:01,306 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/20140bbeabf04269b9806cf14283cc76 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/20140bbeabf04269b9806cf14283cc76 2024-11-27T11:36:01,307 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/3d8a804094f9456099dc7d2e300d4266 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/3d8a804094f9456099dc7d2e300d4266 2024-11-27T11:36:01,308 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/88bd743566364b49b412e0a8b7b605b4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/88bd743566364b49b412e0a8b7b605b4 2024-11-27T11:36:01,309 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4a1dd1f7111f466b858cd98e9e4f6196 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4a1dd1f7111f466b858cd98e9e4f6196 2024-11-27T11:36:01,310 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f20c55261a4844249a7c5387658b74d5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f20c55261a4844249a7c5387658b74d5 2024-11-27T11:36:01,311 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9522d497dad24e70ac6cbe3bfccd87d1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9522d497dad24e70ac6cbe3bfccd87d1 2024-11-27T11:36:01,312 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/19b5e8b9086e4b088646ae0065de8a59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/19b5e8b9086e4b088646ae0065de8a59 2024-11-27T11:36:01,313 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/db1f81f168594040a345a4c4dd5f1b04 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/db1f81f168594040a345a4c4dd5f1b04 2024-11-27T11:36:01,314 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0b1cd520766b419681b37f351d37df12 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0b1cd520766b419681b37f351d37df12 2024-11-27T11:36:01,315 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/586fc0371f21461fbe5cb5e6219d5912 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/586fc0371f21461fbe5cb5e6219d5912 2024-11-27T11:36:01,316 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/939f8799b9af4003b19813629d0d3100 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/939f8799b9af4003b19813629d0d3100 2024-11-27T11:36:01,317 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/09fd59351f6d4cc19aa75d2f1070976f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/09fd59351f6d4cc19aa75d2f1070976f 2024-11-27T11:36:01,318 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/e12924e08078483b95063dc82552979b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/e12924e08078483b95063dc82552979b 2024-11-27T11:36:01,319 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9609a401f209465cb18d83540cd04edd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/9609a401f209465cb18d83540cd04edd 2024-11-27T11:36:01,320 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2ca847c2ef8946e1992884de25aef078 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2ca847c2ef8946e1992884de25aef078 2024-11-27T11:36:01,321 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/056e1a68ab644089b341df5c8257bb83 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/056e1a68ab644089b341df5c8257bb83 2024-11-27T11:36:01,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f67f2bad747d4832888d36f37a3ed5aa to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/f67f2bad747d4832888d36f37a3ed5aa 2024-11-27T11:36:01,323 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4b06526ed1bc484da20b345d14ef9c6a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/4b06526ed1bc484da20b345d14ef9c6a 2024-11-27T11:36:01,324 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0219a63a27cf4fc48a1edced0cbb5640 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/0219a63a27cf4fc48a1edced0cbb5640 2024-11-27T11:36:01,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/d9b4568b52d246feb89997b447b3be09 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/d9b4568b52d246feb89997b447b3be09 2024-11-27T11:36:01,327 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/ab5e6d2f9f774c5d8146fec8323dde12 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/ab5e6d2f9f774c5d8146fec8323dde12 2024-11-27T11:36:01,328 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/887fe7165da54d268678e1b8b4de31b1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/887fe7165da54d268678e1b8b4de31b1 2024-11-27T11:36:01,329 DEBUG [StoreCloser-TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/470a1a8c97a6444990aaafab375d58b1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/470a1a8c97a6444990aaafab375d58b1 2024-11-27T11:36:01,333 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/recovered.edits/382.seqid, newMaxSeqId=382, maxSeqId=4 2024-11-27T11:36:01,334 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. 2024-11-27T11:36:01,334 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for bfc88b634f4291f702daaa0b78cc4f59: Waiting for close lock at 1732707359586Running coprocessor pre-close hooks at 1732707359586Disabling compacts and flushes for region at 1732707359586Disabling writes for close at 1732707359586Obtaining lock to block concurrent updates at 1732707359587 (+1 ms)Preparing flush snapshotting stores in bfc88b634f4291f702daaa0b78cc4f59 at 1732707359587Finished memstore snapshotting TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732707359587Flushing stores of TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59. at 1732707359587Flushing bfc88b634f4291f702daaa0b78cc4f59/A: creating writer at 1732707359587Flushing mob file bfc88b634f4291f702daaa0b78cc4f59/A: appending metadata at 1732707359592 (+5 ms)Flushing mob file bfc88b634f4291f702daaa0b78cc4f59/A: closing flushed file at 1732707359592Flushing bfc88b634f4291f702daaa0b78cc4f59/A: appending metadata at 1732707360002 (+410 ms)Flushing bfc88b634f4291f702daaa0b78cc4f59/A: closing flushed file at 1732707360002Flushing bfc88b634f4291f702daaa0b78cc4f59/B: creating writer at 1732707360410 (+408 ms)Flushing bfc88b634f4291f702daaa0b78cc4f59/B: appending metadata at 1732707360413 (+3 ms)Flushing bfc88b634f4291f702daaa0b78cc4f59/B: closing flushed file at 1732707360413Flushing bfc88b634f4291f702daaa0b78cc4f59/C: creating writer at 1732707360822 (+409 ms)Flushing bfc88b634f4291f702daaa0b78cc4f59/C: appending metadata at 1732707360824 (+2 ms)Flushing bfc88b634f4291f702daaa0b78cc4f59/C: closing flushed file at 1732707360824Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@762e26a0: reopening flushed file at 1732707361233 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14a7db8b: reopening flushed file at 1732707361237 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28b4fe9e: reopening flushed file at 1732707361241 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for bfc88b634f4291f702daaa0b78cc4f59 in 1660ms, sequenceid=379, compaction requested=false at 1732707361246 (+5 ms)Writing region close event to WAL at 1732707361330 (+84 ms)Running coprocessor post-close hooks at 1732707361333 (+3 ms)Closed at 1732707361334 (+1 ms) 2024-11-27T11:36:01,336 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,336 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=bfc88b634f4291f702daaa0b78cc4f59, regionState=CLOSED 2024-11-27T11:36:01,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:36:01,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-27T11:36:01,340 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure bfc88b634f4291f702daaa0b78cc4f59, server=5b617c7edc8e,35461,1732707299005 in 1.9040 sec 2024-11-27T11:36:01,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-27T11:36:01,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bfc88b634f4291f702daaa0b78cc4f59, UNASSIGN in 1.9090 sec 2024-11-27T11:36:01,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-27T11:36:01,344 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9120 sec 2024-11-27T11:36:01,345 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707361345"}]},"ts":"1732707361345"} 2024-11-27T11:36:01,346 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T11:36:01,346 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T11:36:01,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9220 sec 2024-11-27T11:36:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-27T11:36:01,572 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:01,573 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T11:36:01,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,575 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T11:36:01,575 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,578 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,580 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/recovered.edits] 2024-11-27T11:36:01,583 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c873914e79d431db5149b12b32ef2ba to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/1c873914e79d431db5149b12b32ef2ba 2024-11-27T11:36:01,584 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/2f9615459fbc42bb99a6fec00baa971a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/A/2f9615459fbc42bb99a6fec00baa971a 2024-11-27T11:36:01,585 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d6574dd73a3b420abc5f60e4c8a005c8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d6574dd73a3b420abc5f60e4c8a005c8 2024-11-27T11:36:01,586 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d872b0014cff4bb3bb316dae1ade6633 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/B/d872b0014cff4bb3bb316dae1ade6633 2024-11-27T11:36:01,589 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2be48a73765c40ba8d9fc1b55e18c4d5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/2be48a73765c40ba8d9fc1b55e18c4d5 2024-11-27T11:36:01,590 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/bcbf226ebe96448fbe734926a40250b0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/C/bcbf226ebe96448fbe734926a40250b0 2024-11-27T11:36:01,592 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/recovered.edits/382.seqid to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59/recovered.edits/382.seqid 2024-11-27T11:36:01,593 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,593 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T11:36:01,594 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T11:36:01,594 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T11:36:01,598 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271921e8c359c14dce8e34f64be5852c55_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271921e8c359c14dce8e34f64be5852c55_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,599 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271df0f162c93941c38ab6760869ab9eb2_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271df0f162c93941c38ab6760869ab9eb2_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,600 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127210cdd5e0a3e42a0b573246aea1100ce_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127210cdd5e0a3e42a0b573246aea1100ce_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,601 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112726c2c00bbe2744c0b2b2d4794b60eff2_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112726c2c00bbe2744c0b2b2d4794b60eff2_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,602 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127271bc219301d4fbf989e09b700f3a3c1_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127271bc219301d4fbf989e09b700f3a3c1_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,603 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112729d35976ab464024969a08d26a6bdb38_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112729d35976ab464024969a08d26a6bdb38_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112767b127815f3e448eb68bdc65ff01dfb4_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112767b127815f3e448eb68bdc65ff01dfb4_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,605 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127704932bbaf3a4bd4856be8e7776d0c06_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127704932bbaf3a4bd4856be8e7776d0c06_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,607 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c0681439dcd4dabbab83d56d0712fc9_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411277c0681439dcd4dabbab83d56d0712fc9_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127801fc237cc004ab4bc8eef0b249b796d_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127801fc237cc004ab4bc8eef0b249b796d_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,609 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783dc026f3b2043aca2d85ad9f8b6a1c5_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112783dc026f3b2043aca2d85ad9f8b6a1c5_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,610 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278403ed9a9b2c440ca1b850eea255d9a0_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278403ed9a9b2c440ca1b850eea255d9a0_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,612 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787b49b0ce89446198dfd1c0e558f8c2c_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787b49b0ce89446198dfd1c0e558f8c2c_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,613 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279a075929f9654a89ae5a11d3a100d9a1_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279a075929f9654a89ae5a11d3a100d9a1_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,614 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279cafd3bc0e0b4233a8fba63e12e01da5_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279cafd3bc0e0b4233a8fba63e12e01da5_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,615 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0d9bba24d0b4f59b2d96289c0d9b7a7_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d0d9bba24d0b4f59b2d96289c0d9b7a7_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,616 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dada4e7eed4241efbe6d38d841625d83_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dada4e7eed4241efbe6d38d841625d83_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,617 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eceaa2edbc03472e92f6530f2b1ef054_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eceaa2edbc03472e92f6530f2b1ef054_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,618 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3a7b084558e4ee0b44abbb410b2905b_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3a7b084558e4ee0b44abbb410b2905b_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f788ef6ac7b54f50ad9110e05d4a0c3f_bfc88b634f4291f702daaa0b78cc4f59 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f788ef6ac7b54f50ad9110e05d4a0c3f_bfc88b634f4291f702daaa0b78cc4f59 2024-11-27T11:36:01,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T11:36:01,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,624 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T11:36:01,626 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T11:36:01,627 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,627 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T11:36:01,627 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732707361627"}]},"ts":"9223372036854775807"} 2024-11-27T11:36:01,629 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T11:36:01,629 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bfc88b634f4291f702daaa0b78cc4f59, NAME => 'TestAcidGuarantees,,1732707328795.bfc88b634f4291f702daaa0b78cc4f59.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T11:36:01,630 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T11:36:01,630 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732707361630"}]},"ts":"9223372036854775807"} 2024-11-27T11:36:01,631 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T11:36:01,632 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,633 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 59 msec 2024-11-27T11:36:01,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-27T11:36:01,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T11:36:01,682 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:01,692 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 238), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 327), ProcessCount=11 (was 11), AvailableMemoryMB=6443 (was 6541) 2024-11-27T11:36:01,700 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=301, ProcessCount=11, AvailableMemoryMB=6443 2024-11-27T11:36:01,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:36:01,702 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:36:01,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:01,704 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T11:36:01,704 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:01,704 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-27T11:36:01,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T11:36:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-27T11:36:01,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742061_1237 (size=959) 2024-11-27T11:36:01,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-27T11:36:02,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-27T11:36:02,112 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:36:02,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742062_1238 (size=53) 2024-11-27T11:36:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-27T11:36:02,518 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:36:02,519 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5e109c5702a251266cfb057894b39f84, disabling compactions & flushes 2024-11-27T11:36:02,519 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,519 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,519 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. after waiting 0 ms 2024-11-27T11:36:02,519 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,519 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,519 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5e109c5702a251266cfb057894b39f84: Waiting for close lock at 1732707362519Disabling compacts and flushes for region at 1732707362519Disabling writes for close at 1732707362519Writing region close event to WAL at 1732707362519Closed at 1732707362519 2024-11-27T11:36:02,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T11:36:02,520 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732707362520"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732707362520"}]},"ts":"1732707362520"} 2024-11-27T11:36:02,522 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T11:36:02,523 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T11:36:02,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707362523"}]},"ts":"1732707362523"} 2024-11-27T11:36:02,524 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T11:36:02,524 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, ASSIGN}] 2024-11-27T11:36:02,525 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, ASSIGN 2024-11-27T11:36:02,526 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, ASSIGN; state=OFFLINE, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=false 2024-11-27T11:36:02,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5e109c5702a251266cfb057894b39f84, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:02,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, ASSIGN because future has completed 2024-11-27T11:36:02,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:36:02,833 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,833 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:36:02,834 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,834 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:36:02,834 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,834 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,835 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,836 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e109c5702a251266cfb057894b39f84 columnFamilyName A 2024-11-27T11:36:02,836 DEBUG [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:02,837 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] regionserver.HStore(327): Store=5e109c5702a251266cfb057894b39f84/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:02,837 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,838 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e109c5702a251266cfb057894b39f84 columnFamilyName B 2024-11-27T11:36:02,838 DEBUG [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:02,838 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] regionserver.HStore(327): Store=5e109c5702a251266cfb057894b39f84/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:02,839 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,840 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5e109c5702a251266cfb057894b39f84 columnFamilyName C 2024-11-27T11:36:02,840 DEBUG [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:02,840 INFO [StoreOpener-5e109c5702a251266cfb057894b39f84-1 {}] regionserver.HStore(327): Store=5e109c5702a251266cfb057894b39f84/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:02,840 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,841 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,841 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,842 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,842 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,842 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:36:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-27T11:36:02,843 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,845 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:36:02,846 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 5e109c5702a251266cfb057894b39f84; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70355343, jitterRate=0.048376306891441345}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:36:02,846 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:02,846 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 5e109c5702a251266cfb057894b39f84: Running coprocessor pre-open hook at 1732707362834Writing region info on filesystem at 1732707362834Initializing all the Stores at 1732707362835 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707362835Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707362835Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707362835Cleaning up temporary data from old regions at 1732707362842 (+7 ms)Running coprocessor post-open hooks at 1732707362846 (+4 ms)Region opened successfully at 1732707362846 2024-11-27T11:36:02,847 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., pid=63, masterSystemTime=1732707362830 2024-11-27T11:36:02,849 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,849 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:02,849 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=5e109c5702a251266cfb057894b39f84, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:02,850 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:36:02,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-27T11:36:02,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 in 172 msec 2024-11-27T11:36:02,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-27T11:36:02,854 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, ASSIGN in 328 msec 2024-11-27T11:36:02,855 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T11:36:02,855 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707362855"}]},"ts":"1732707362855"} 2024-11-27T11:36:02,856 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T11:36:02,872 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T11:36:02,874 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-27T11:36:03,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-27T11:36:03,863 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:03,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60a7e197, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,865 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32ebcf80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,867 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,867 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bab3a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,868 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,868 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,870 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:36:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:36:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79a86dc5 2024-11-27T11:36:03,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:36:03,872 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:36:03,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f42fbd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@104a8300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b2ddc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491abcef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,875 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,876 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60475b56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,877 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13939afe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740e5304, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,878 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,879 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541d8129, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,880 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60698, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f9f2a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5bd173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,881 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,881 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15c732d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,883 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d79529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,883 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57a6b702, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,883 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,883 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,884 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fac2b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,885 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20464335, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,886 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f909286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,886 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,886 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,887 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7041cbe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,887 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,888 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bb9396f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,889 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4733e7c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,889 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,889 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,889 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@351db927, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,890 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e3184b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e2df343, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,891 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,891 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,892 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1929c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,893 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564f4474, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,894 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@313437ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,894 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,894 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,894 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad72e24, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,894 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,895 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,896 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25e4959d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@306c034b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:03,897 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:03,897 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50baf510, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,897 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:03,898 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:03,898 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60810, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:03,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5531708f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:03,899 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,899 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,899 DEBUG [Thread-1115 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:03,900 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,901 DEBUG [Thread-1109 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,901 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57266, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57294, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57304, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57278, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,902 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,902 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,903 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,903 DEBUG [Thread-1111 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,903 DEBUG [Thread-1120 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,905 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57346, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-27T11:36:03,905 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,906 DEBUG [Thread-1113 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,906 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,906 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,908 DEBUG [Thread-1117 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:03,908 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:03,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,908 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:03,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:03,909 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,910 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:03,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:03,912 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:03,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:36:03,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:03,965 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:03,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:36:03,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/0ff8ae235392406eb45186550da97fa6 is 50, key is test_row_0/A:col10/1732707363922/Put/seqid=0 2024-11-27T11:36:03,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:03,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707423993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707423994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707423996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,000 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707423998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707423999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742063_1239 (size=12001) 2024-11-27T11:36:04,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:04,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707424117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707424117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707424117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,119 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707424117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707424120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:04,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707424322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707424322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,325 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707424322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707424333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707424333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,334 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:04,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/0ff8ae235392406eb45186550da97fa6 2024-11-27T11:36:04,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/5c10a26b5162464988a8301c252cff4c is 50, key is test_row_0/B:col10/1732707363922/Put/seqid=0 2024-11-27T11:36:04,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742064_1240 (size=12001) 2024-11-27T11:36:04,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:04,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:04,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707424633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707424633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707424633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707424642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707424642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:04,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:04,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:04,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:04,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:04,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/5c10a26b5162464988a8301c252cff4c 2024-11-27T11:36:04,863 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/08752b1a91054eb98e3897fe437f8dc0 is 50, key is test_row_0/C:col10/1732707363922/Put/seqid=0 2024-11-27T11:36:04,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742065_1241 (size=12001) 2024-11-27T11:36:04,868 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/08752b1a91054eb98e3897fe437f8dc0 2024-11-27T11:36:04,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/0ff8ae235392406eb45186550da97fa6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/0ff8ae235392406eb45186550da97fa6 2024-11-27T11:36:04,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/0ff8ae235392406eb45186550da97fa6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T11:36:04,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/5c10a26b5162464988a8301c252cff4c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/5c10a26b5162464988a8301c252cff4c 2024-11-27T11:36:04,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/5c10a26b5162464988a8301c252cff4c, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T11:36:04,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/08752b1a91054eb98e3897fe437f8dc0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/08752b1a91054eb98e3897fe437f8dc0 2024-11-27T11:36:04,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/08752b1a91054eb98e3897fe437f8dc0, entries=150, sequenceid=14, filesize=11.7 K 2024-11-27T11:36:04,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 5e109c5702a251266cfb057894b39f84 in 923ms, sequenceid=14, compaction requested=false 2024-11-27T11:36:04,888 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-27T11:36:04,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:04,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-27T11:36:04,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:04,989 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:36:04,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e6bf16a33c9c4c92955f18025102c857 is 50, key is test_row_0/A:col10/1732707363992/Put/seqid=0 2024-11-27T11:36:04,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742066_1242 (size=12001) 2024-11-27T11:36:05,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:05,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:05,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707425153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707425154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707425157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707425158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707425158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707425262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707425262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707425273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707425273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707425273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,399 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e6bf16a33c9c4c92955f18025102c857 2024-11-27T11:36:05,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/deadf7b2bac448a8b44e13562dc55f4a is 50, key is test_row_0/B:col10/1732707363992/Put/seqid=0 2024-11-27T11:36:05,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742067_1243 (size=12001) 2024-11-27T11:36:05,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707425472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707425473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707425482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707425482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707425482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707425782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707425782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707425793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707425793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707425793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:05,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:05,812 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/deadf7b2bac448a8b44e13562dc55f4a 2024-11-27T11:36:05,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/b738e10da49b4e15bb1452e99727ecf0 is 50, key is test_row_0/C:col10/1732707363992/Put/seqid=0 2024-11-27T11:36:05,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742068_1244 (size=12001) 2024-11-27T11:36:06,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:06,226 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/b738e10da49b4e15bb1452e99727ecf0 2024-11-27T11:36:06,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e6bf16a33c9c4c92955f18025102c857 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e6bf16a33c9c4c92955f18025102c857 2024-11-27T11:36:06,235 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e6bf16a33c9c4c92955f18025102c857, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T11:36:06,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/deadf7b2bac448a8b44e13562dc55f4a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/deadf7b2bac448a8b44e13562dc55f4a 2024-11-27T11:36:06,241 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/deadf7b2bac448a8b44e13562dc55f4a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T11:36:06,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/b738e10da49b4e15bb1452e99727ecf0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/b738e10da49b4e15bb1452e99727ecf0 2024-11-27T11:36:06,249 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/b738e10da49b4e15bb1452e99727ecf0, entries=150, sequenceid=38, filesize=11.7 K 2024-11-27T11:36:06,249 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 5e109c5702a251266cfb057894b39f84 in 1260ms, sequenceid=38, compaction requested=false 2024-11-27T11:36:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-27T11:36:06,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-27T11:36:06,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-27T11:36:06,254 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3400 sec 2024-11-27T11:36:06,256 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.3550 sec 2024-11-27T11:36:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:06,296 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:06,296 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T11:36:06,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/9d468fb344cb4009b3ddb9a060f6ec5c is 50, key is test_row_0/A:col10/1732707365156/Put/seqid=0 2024-11-27T11:36:06,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742069_1245 (size=16681) 2024-11-27T11:36:06,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707426323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707426324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707426325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707426326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707426326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707426432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,434 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707426433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707426433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707426433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707426442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707426642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707426642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707426643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707426643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707426653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/9d468fb344cb4009b3ddb9a060f6ec5c 2024-11-27T11:36:06,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/d26c7e5d0ed3496d82913e3345fc041d is 50, key is test_row_0/B:col10/1732707365156/Put/seqid=0 2024-11-27T11:36:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742070_1246 (size=12001) 2024-11-27T11:36:06,864 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:36:06,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707426952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707426952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707426952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707426952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:06,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:06,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707426962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:06,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:06,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:07,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/d26c7e5d0ed3496d82913e3345fc041d 2024-11-27T11:36:07,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/60b967beec3049559485a47594339ee7 is 50, key is test_row_0/C:col10/1732707365156/Put/seqid=0 2024-11-27T11:36:07,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742071_1247 (size=12001) 2024-11-27T11:36:07,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707427462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707427462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:07,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707427462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:07,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707427462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:07,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:07,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707427473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:07,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:07,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/60b967beec3049559485a47594339ee7 2024-11-27T11:36:07,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/9d468fb344cb4009b3ddb9a060f6ec5c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9d468fb344cb4009b3ddb9a060f6ec5c 2024-11-27T11:36:07,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9d468fb344cb4009b3ddb9a060f6ec5c, entries=250, sequenceid=53, filesize=16.3 K 2024-11-27T11:36:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/d26c7e5d0ed3496d82913e3345fc041d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d26c7e5d0ed3496d82913e3345fc041d 2024-11-27T11:36:07,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d26c7e5d0ed3496d82913e3345fc041d, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T11:36:07,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/60b967beec3049559485a47594339ee7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/60b967beec3049559485a47594339ee7 2024-11-27T11:36:07,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/60b967beec3049559485a47594339ee7, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T11:36:07,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 5e109c5702a251266cfb057894b39f84 in 1264ms, sequenceid=53, compaction requested=true 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:07,560 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:07,560 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:07,561 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:07,561 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:07,561 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:07,561 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:07,561 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:07,561 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:07,561 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/0ff8ae235392406eb45186550da97fa6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e6bf16a33c9c4c92955f18025102c857, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9d468fb344cb4009b3ddb9a060f6ec5c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=39.7 K 2024-11-27T11:36:07,561 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/5c10a26b5162464988a8301c252cff4c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/deadf7b2bac448a8b44e13562dc55f4a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d26c7e5d0ed3496d82913e3345fc041d] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.2 K 2024-11-27T11:36:07,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ff8ae235392406eb45186550da97fa6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732707363913 2024-11-27T11:36:07,562 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c10a26b5162464988a8301c252cff4c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732707363913 2024-11-27T11:36:07,562 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6bf16a33c9c4c92955f18025102c857, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732707363992 2024-11-27T11:36:07,563 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting deadf7b2bac448a8b44e13562dc55f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732707363992 2024-11-27T11:36:07,563 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d468fb344cb4009b3ddb9a060f6ec5c, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707365156 2024-11-27T11:36:07,563 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d26c7e5d0ed3496d82913e3345fc041d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707365156 2024-11-27T11:36:07,575 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#203 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:07,575 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:07,576 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/81f6c986bf9843879f4a76317171ff19 is 50, key is test_row_0/B:col10/1732707365156/Put/seqid=0 2024-11-27T11:36:07,576 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/ff29028f13484b9f8ca792ed78b9a89f is 50, key is test_row_0/A:col10/1732707365156/Put/seqid=0 2024-11-27T11:36:07,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742072_1248 (size=12104) 2024-11-27T11:36:07,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742073_1249 (size=12104) 2024-11-27T11:36:07,596 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/81f6c986bf9843879f4a76317171ff19 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/81f6c986bf9843879f4a76317171ff19 2024-11-27T11:36:07,601 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into 81f6c986bf9843879f4a76317171ff19(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:07,601 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:07,601 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=13, startTime=1732707367560; duration=0sec 2024-11-27T11:36:07,601 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:07,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:07,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:07,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:07,603 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:07,603 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:07,603 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/08752b1a91054eb98e3897fe437f8dc0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/b738e10da49b4e15bb1452e99727ecf0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/60b967beec3049559485a47594339ee7] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.2 K 2024-11-27T11:36:07,603 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 08752b1a91054eb98e3897fe437f8dc0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732707363913 2024-11-27T11:36:07,604 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b738e10da49b4e15bb1452e99727ecf0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732707363992 2024-11-27T11:36:07,604 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b967beec3049559485a47594339ee7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707365156 2024-11-27T11:36:07,616 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:07,616 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/3851716dc0ae47828b5d81d6419b4b7a is 50, key is test_row_0/C:col10/1732707365156/Put/seqid=0 2024-11-27T11:36:07,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742074_1250 (size=12104) 2024-11-27T11:36:07,632 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/3851716dc0ae47828b5d81d6419b4b7a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/3851716dc0ae47828b5d81d6419b4b7a 2024-11-27T11:36:07,639 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into 3851716dc0ae47828b5d81d6419b4b7a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:07,640 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:07,640 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=13, startTime=1732707367560; duration=0sec 2024-11-27T11:36:07,640 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:07,640 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:07,988 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/ff29028f13484b9f8ca792ed78b9a89f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/ff29028f13484b9f8ca792ed78b9a89f 2024-11-27T11:36:07,993 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into ff29028f13484b9f8ca792ed78b9a89f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:07,993 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:07,993 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=13, startTime=1732707367560; duration=0sec 2024-11-27T11:36:07,993 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:07,993 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-27T11:36:08,062 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:08,063 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-27T11:36:08,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T11:36:08,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:08,065 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:08,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:08,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T11:36:08,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-27T11:36:08,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:08,219 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T11:36:08,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/9dcf4a7709be4300b2c3d9fac05dabd6 is 50, key is test_row_0/A:col10/1732707366321/Put/seqid=0 2024-11-27T11:36:08,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742075_1251 (size=12001) 2024-11-27T11:36:08,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T11:36:08,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:08,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:08,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707428487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707428519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707428519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707428519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707428520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707428622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707428632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707428632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707428633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707428633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,643 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/9dcf4a7709be4300b2c3d9fac05dabd6 2024-11-27T11:36:08,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/4c775b23cee643c4b5996ede370b1148 is 50, key is test_row_0/B:col10/1732707366321/Put/seqid=0 2024-11-27T11:36:08,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742076_1252 (size=12001) 2024-11-27T11:36:08,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T11:36:08,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707428832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,834 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707428843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707428843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707428843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:08,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:08,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707428843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:08,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,059 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/4c775b23cee643c4b5996ede370b1148 2024-11-27T11:36:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/7e7fc3cc1afc412b8d98b6022daa01c4 is 50, key is test_row_0/C:col10/1732707366321/Put/seqid=0 2024-11-27T11:36:09,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742077_1253 (size=12001) 2024-11-27T11:36:09,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707429143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707429152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707429152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707429152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707429153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T11:36:09,481 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/7e7fc3cc1afc412b8d98b6022daa01c4 2024-11-27T11:36:09,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/9dcf4a7709be4300b2c3d9fac05dabd6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9dcf4a7709be4300b2c3d9fac05dabd6 2024-11-27T11:36:09,491 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9dcf4a7709be4300b2c3d9fac05dabd6, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T11:36:09,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/4c775b23cee643c4b5996ede370b1148 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/4c775b23cee643c4b5996ede370b1148 2024-11-27T11:36:09,496 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/4c775b23cee643c4b5996ede370b1148, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T11:36:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/7e7fc3cc1afc412b8d98b6022daa01c4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7e7fc3cc1afc412b8d98b6022daa01c4 2024-11-27T11:36:09,500 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7e7fc3cc1afc412b8d98b6022daa01c4, entries=150, sequenceid=78, filesize=11.7 K 2024-11-27T11:36:09,501 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 5e109c5702a251266cfb057894b39f84 in 1283ms, sequenceid=78, compaction requested=false 2024-11-27T11:36:09,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:09,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-27T11:36:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-27T11:36:09,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-27T11:36:09,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4370 sec 2024-11-27T11:36:09,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.4430 sec 2024-11-27T11:36:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:09,654 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:09,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T11:36:09,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/60bc9e27c0aa4803a33beee3a346868a is 50, key is test_row_0/A:col10/1732707368493/Put/seqid=0 2024-11-27T11:36:09,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742078_1254 (size=14341) 2024-11-27T11:36:09,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707429677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,681 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707429679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707429679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707429681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707429681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707429792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707429792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707429792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707429792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:09,794 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707429792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:09,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707430002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707430002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707430002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707430002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707430002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/60bc9e27c0aa4803a33beee3a346868a 2024-11-27T11:36:10,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/17b3c1cac242496c906150fd8f39afb1 is 50, key is test_row_0/B:col10/1732707368493/Put/seqid=0 2024-11-27T11:36:10,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742079_1255 (size=12001) 2024-11-27T11:36:10,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/17b3c1cac242496c906150fd8f39afb1 2024-11-27T11:36:10,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/c32c36e4377046a281e3efbd08cccdd2 is 50, key is test_row_0/C:col10/1732707368493/Put/seqid=0 2024-11-27T11:36:10,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742080_1256 (size=12001) 2024-11-27T11:36:10,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-27T11:36:10,212 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:10,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:10,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-27T11:36:10,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T11:36:10,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:10,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:10,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:10,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707430312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707430312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707430312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707430312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707430312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T11:36:10,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T11:36:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:10,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:10,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:10,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:10,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:10,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/c32c36e4377046a281e3efbd08cccdd2 2024-11-27T11:36:10,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/60bc9e27c0aa4803a33beee3a346868a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/60bc9e27c0aa4803a33beee3a346868a 2024-11-27T11:36:10,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/60bc9e27c0aa4803a33beee3a346868a, entries=200, sequenceid=94, filesize=14.0 K 2024-11-27T11:36:10,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/17b3c1cac242496c906150fd8f39afb1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/17b3c1cac242496c906150fd8f39afb1 2024-11-27T11:36:10,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/17b3c1cac242496c906150fd8f39afb1, entries=150, sequenceid=94, filesize=11.7 K 2024-11-27T11:36:10,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/c32c36e4377046a281e3efbd08cccdd2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/c32c36e4377046a281e3efbd08cccdd2 2024-11-27T11:36:10,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/c32c36e4377046a281e3efbd08cccdd2, entries=150, sequenceid=94, filesize=11.7 K 2024-11-27T11:36:10,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5e109c5702a251266cfb057894b39f84 in 863ms, sequenceid=94, compaction requested=true 2024-11-27T11:36:10,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:10,517 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:10,518 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:10,518 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:10,519 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:10,519 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:10,519 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:10,519 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:10,519 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:10,519 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:10,519 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/81f6c986bf9843879f4a76317171ff19, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/4c775b23cee643c4b5996ede370b1148, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/17b3c1cac242496c906150fd8f39afb1] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.3 K 2024-11-27T11:36:10,519 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/ff29028f13484b9f8ca792ed78b9a89f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9dcf4a7709be4300b2c3d9fac05dabd6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/60bc9e27c0aa4803a33beee3a346868a] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=37.5 K 2024-11-27T11:36:10,520 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81f6c986bf9843879f4a76317171ff19, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707365156 2024-11-27T11:36:10,520 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ff29028f13484b9f8ca792ed78b9a89f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707365156 2024-11-27T11:36:10,520 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c775b23cee643c4b5996ede370b1148, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732707366321 2024-11-27T11:36:10,520 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcf4a7709be4300b2c3d9fac05dabd6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732707366321 2024-11-27T11:36:10,521 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b3c1cac242496c906150fd8f39afb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732707368493 2024-11-27T11:36:10,521 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 60bc9e27c0aa4803a33beee3a346868a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732707368492 2024-11-27T11:36:10,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-27T11:36:10,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:10,525 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T11:36:10,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T11:36:10,550 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:10,551 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/f2937869f381488ba8441c3fc105612f is 50, key is test_row_0/B:col10/1732707368493/Put/seqid=0 2024-11-27T11:36:10,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/67fd6509ad9b46198cedf50432c1a7b4 is 50, key is test_row_0/A:col10/1732707369679/Put/seqid=0 2024-11-27T11:36:10,567 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#213 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:10,568 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e034db997fdd429cbe452f05dc02cc6c is 50, key is test_row_0/A:col10/1732707368493/Put/seqid=0 2024-11-27T11:36:10,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742081_1257 (size=12207) 2024-11-27T11:36:10,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742083_1259 (size=12207) 2024-11-27T11:36:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742082_1258 (size=12001) 2024-11-27T11:36:10,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:10,825 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:10,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707430836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707430837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T11:36:10,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707430838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707430838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,844 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707430841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707430952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707430952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707430952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707430952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707430952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:10,989 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/f2937869f381488ba8441c3fc105612f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/f2937869f381488ba8441c3fc105612f 2024-11-27T11:36:10,991 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/67fd6509ad9b46198cedf50432c1a7b4 2024-11-27T11:36:10,999 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into f2937869f381488ba8441c3fc105612f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:10,999 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:10,999 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=13, startTime=1732707370518; duration=0sec 2024-11-27T11:36:10,999 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:10,999 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:11,000 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:11,003 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e034db997fdd429cbe452f05dc02cc6c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e034db997fdd429cbe452f05dc02cc6c 2024-11-27T11:36:11,004 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:11,009 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:11,009 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:11,009 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/3851716dc0ae47828b5d81d6419b4b7a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7e7fc3cc1afc412b8d98b6022daa01c4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/c32c36e4377046a281e3efbd08cccdd2] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.3 K 2024-11-27T11:36:11,010 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3851716dc0ae47828b5d81d6419b4b7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707365156 2024-11-27T11:36:11,010 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e7fc3cc1afc412b8d98b6022daa01c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732707366321 2024-11-27T11:36:11,011 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into e034db997fdd429cbe452f05dc02cc6c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:11,011 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:11,011 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=13, startTime=1732707370517; duration=0sec 2024-11-27T11:36:11,011 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c32c36e4377046a281e3efbd08cccdd2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732707368493 2024-11-27T11:36:11,011 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:11,011 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:11,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/35f5f05992ed4803b8f5a8addebd5c5f is 50, key is test_row_0/B:col10/1732707369679/Put/seqid=0 2024-11-27T11:36:11,034 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#215 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:11,035 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/7267326b54fd4662b40f66751f42f0b2 is 50, key is test_row_0/C:col10/1732707368493/Put/seqid=0 2024-11-27T11:36:11,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742084_1260 (size=12001) 2024-11-27T11:36:11,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742085_1261 (size=12207) 2024-11-27T11:36:11,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707431163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707431163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707431163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,165 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707431163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707431163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T11:36:11,443 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/35f5f05992ed4803b8f5a8addebd5c5f 2024-11-27T11:36:11,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/d418e822a22c4e0190b19c2987901cfc is 50, key is test_row_0/C:col10/1732707369679/Put/seqid=0 2024-11-27T11:36:11,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742086_1262 (size=12001) 2024-11-27T11:36:11,465 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/7267326b54fd4662b40f66751f42f0b2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7267326b54fd4662b40f66751f42f0b2 2024-11-27T11:36:11,469 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into 7267326b54fd4662b40f66751f42f0b2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:11,469 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:11,469 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=13, startTime=1732707370518; duration=0sec 2024-11-27T11:36:11,469 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:11,469 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:11,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707431472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707431472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707431472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707431473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707431473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:11,477 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:11,859 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/d418e822a22c4e0190b19c2987901cfc 2024-11-27T11:36:11,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/67fd6509ad9b46198cedf50432c1a7b4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/67fd6509ad9b46198cedf50432c1a7b4 2024-11-27T11:36:11,868 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/67fd6509ad9b46198cedf50432c1a7b4, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T11:36:11,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/35f5f05992ed4803b8f5a8addebd5c5f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/35f5f05992ed4803b8f5a8addebd5c5f 2024-11-27T11:36:11,872 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/35f5f05992ed4803b8f5a8addebd5c5f, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T11:36:11,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/d418e822a22c4e0190b19c2987901cfc as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d418e822a22c4e0190b19c2987901cfc 2024-11-27T11:36:11,877 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d418e822a22c4e0190b19c2987901cfc, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T11:36:11,878 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 5e109c5702a251266cfb057894b39f84 in 1353ms, sequenceid=114, compaction requested=false 2024-11-27T11:36:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:11,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-27T11:36:11,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-27T11:36:11,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-27T11:36:11,881 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-27T11:36:11,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.6690 sec 2024-11-27T11:36:11,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:11,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:11,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-27T11:36:11,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/b0f5bdb314ac4ccc913f3227ebfee595 is 50, key is test_row_0/A:col10/1732707370839/Put/seqid=0 2024-11-27T11:36:12,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742087_1263 (size=12101) 2024-11-27T11:36:12,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707432006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707432007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707432007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707432010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,014 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707432013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707432122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707432123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707432123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707432123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707432123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707432332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707432333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707432333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707432333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707432333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-27T11:36:12,363 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:12,363 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:12,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-27T11:36:12,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T11:36:12,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:12,366 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:12,366 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:12,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/b0f5bdb314ac4ccc913f3227ebfee595 2024-11-27T11:36:12,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/b304ac20701049f29d1625250ca19f57 is 50, key is test_row_0/B:col10/1732707370839/Put/seqid=0 2024-11-27T11:36:12,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742088_1264 (size=12101) 2024-11-27T11:36:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T11:36:12,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T11:36:12,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:12,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707432642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707432643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707432643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707432643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:12,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707432643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:12,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:12,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T11:36:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T11:36:12,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/b304ac20701049f29d1625250ca19f57 2024-11-27T11:36:12,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T11:36:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:12,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/9e7bcc46612842ecb6f31d0b2bee5bf4 is 50, key is test_row_0/C:col10/1732707370839/Put/seqid=0 2024-11-27T11:36:12,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742089_1265 (size=12101) 2024-11-27T11:36:12,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/9e7bcc46612842ecb6f31d0b2bee5bf4 2024-11-27T11:36:12,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/b0f5bdb314ac4ccc913f3227ebfee595 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b0f5bdb314ac4ccc913f3227ebfee595 2024-11-27T11:36:12,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b0f5bdb314ac4ccc913f3227ebfee595, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T11:36:12,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/b304ac20701049f29d1625250ca19f57 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b304ac20701049f29d1625250ca19f57 2024-11-27T11:36:12,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b304ac20701049f29d1625250ca19f57, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T11:36:12,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/9e7bcc46612842ecb6f31d0b2bee5bf4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/9e7bcc46612842ecb6f31d0b2bee5bf4 2024-11-27T11:36:12,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/9e7bcc46612842ecb6f31d0b2bee5bf4, entries=150, sequenceid=135, filesize=11.8 K 2024-11-27T11:36:12,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 5e109c5702a251266cfb057894b39f84 in 876ms, sequenceid=135, compaction requested=true 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:12,860 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:12,860 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:12,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:12,861 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:12,861 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:12,862 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,862 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e034db997fdd429cbe452f05dc02cc6c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/67fd6509ad9b46198cedf50432c1a7b4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b0f5bdb314ac4ccc913f3227ebfee595] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.5 K 2024-11-27T11:36:12,862 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:12,862 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e034db997fdd429cbe452f05dc02cc6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732707368493 2024-11-27T11:36:12,862 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:12,862 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,862 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/f2937869f381488ba8441c3fc105612f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/35f5f05992ed4803b8f5a8addebd5c5f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b304ac20701049f29d1625250ca19f57] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.5 K 2024-11-27T11:36:12,862 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67fd6509ad9b46198cedf50432c1a7b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732707369676 2024-11-27T11:36:12,863 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f2937869f381488ba8441c3fc105612f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732707368493 2024-11-27T11:36:12,863 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0f5bdb314ac4ccc913f3227ebfee595, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732707370835 2024-11-27T11:36:12,863 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f5f05992ed4803b8f5a8addebd5c5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732707369676 2024-11-27T11:36:12,863 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b304ac20701049f29d1625250ca19f57, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732707370835 2024-11-27T11:36:12,872 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#220 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:12,872 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#221 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:12,873 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e3539f76ccba416c8f60a412ba15e08a is 50, key is test_row_0/A:col10/1732707370839/Put/seqid=0 2024-11-27T11:36:12,873 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/908595be150b4b878584a74e1b04a711 is 50, key is test_row_0/B:col10/1732707370839/Put/seqid=0 2024-11-27T11:36:12,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742090_1266 (size=12409) 2024-11-27T11:36:12,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742091_1267 (size=12409) 2024-11-27T11:36:12,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-27T11:36:12,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:12,979 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-27T11:36:12,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/6b8684f94e1a423e8e1adc78f74159ba is 50, key is test_row_0/A:col10/1732707372008/Put/seqid=0 2024-11-27T11:36:12,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T11:36:13,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742092_1268 (size=12151) 2024-11-27T11:36:13,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:13,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:13,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707433167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707433169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,174 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707433170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707433172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707433172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,285 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e3539f76ccba416c8f60a412ba15e08a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e3539f76ccba416c8f60a412ba15e08a 2024-11-27T11:36:13,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707433283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707433283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707433283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707433283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707433283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,290 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into e3539f76ccba416c8f60a412ba15e08a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:13,290 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:13,290 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=13, startTime=1732707372860; duration=0sec 2024-11-27T11:36:13,290 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:13,290 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:13,290 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:13,291 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:13,291 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:13,291 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:13,291 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7267326b54fd4662b40f66751f42f0b2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d418e822a22c4e0190b19c2987901cfc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/9e7bcc46612842ecb6f31d0b2bee5bf4] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.5 K 2024-11-27T11:36:13,292 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7267326b54fd4662b40f66751f42f0b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732707368493 2024-11-27T11:36:13,292 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d418e822a22c4e0190b19c2987901cfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732707369676 2024-11-27T11:36:13,293 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e7bcc46612842ecb6f31d0b2bee5bf4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732707370835 2024-11-27T11:36:13,295 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/908595be150b4b878584a74e1b04a711 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/908595be150b4b878584a74e1b04a711 2024-11-27T11:36:13,299 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into 908595be150b4b878584a74e1b04a711(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:13,300 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:13,300 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=13, startTime=1732707372860; duration=0sec 2024-11-27T11:36:13,300 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:13,300 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:13,300 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#223 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:13,301 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/2606bb3776d34f2aa610b843632fad5f is 50, key is test_row_0/C:col10/1732707370839/Put/seqid=0 2024-11-27T11:36:13,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742093_1269 (size=12409) 2024-11-27T11:36:13,411 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/6b8684f94e1a423e8e1adc78f74159ba 2024-11-27T11:36:13,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/a242a5e699414786a664131d7878c7af is 50, key is test_row_0/B:col10/1732707372008/Put/seqid=0 2024-11-27T11:36:13,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742094_1270 (size=12151) 2024-11-27T11:36:13,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707433493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707433493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707433493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707433493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707433493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T11:36:13,710 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/2606bb3776d34f2aa610b843632fad5f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/2606bb3776d34f2aa610b843632fad5f 2024-11-27T11:36:13,715 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into 2606bb3776d34f2aa610b843632fad5f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:13,715 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:13,715 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=13, startTime=1732707372860; duration=0sec 2024-11-27T11:36:13,715 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:13,715 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707433802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707433802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707433803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,806 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707433803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:13,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707433803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:13,807 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:13,827 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/a242a5e699414786a664131d7878c7af 2024-11-27T11:36:13,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/ee24fcc9a16f4097942957db48717dae is 50, key is test_row_0/C:col10/1732707372008/Put/seqid=0 2024-11-27T11:36:13,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742095_1271 (size=12151) 2024-11-27T11:36:13,842 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/ee24fcc9a16f4097942957db48717dae 2024-11-27T11:36:13,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/6b8684f94e1a423e8e1adc78f74159ba as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/6b8684f94e1a423e8e1adc78f74159ba 2024-11-27T11:36:13,855 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/6b8684f94e1a423e8e1adc78f74159ba, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T11:36:13,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/a242a5e699414786a664131d7878c7af as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a242a5e699414786a664131d7878c7af 2024-11-27T11:36:13,860 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a242a5e699414786a664131d7878c7af, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T11:36:13,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/ee24fcc9a16f4097942957db48717dae as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ee24fcc9a16f4097942957db48717dae 2024-11-27T11:36:13,864 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ee24fcc9a16f4097942957db48717dae, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T11:36:13,865 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 5e109c5702a251266cfb057894b39f84 in 886ms, sequenceid=153, compaction requested=false 2024-11-27T11:36:13,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:13,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-27T11:36:13,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-27T11:36:13,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-27T11:36:13,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5010 sec 2024-11-27T11:36:13,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5070 sec 2024-11-27T11:36:14,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:14,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:14,315 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-27T11:36:14,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/b2dd94d743ff48409839b2564606f5e9 is 50, key is test_row_0/A:col10/1732707373170/Put/seqid=0 2024-11-27T11:36:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742096_1272 (size=12151) 2024-11-27T11:36:14,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707434327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707434331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707434333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707434334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707434334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707434442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707434442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707434443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707434443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707434443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,447 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-27T11:36:14,513 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:14,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-27T11:36:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T11:36:14,515 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:14,516 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:14,516 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T11:36:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707434653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707434653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707434653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707434653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707434653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:14,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/b2dd94d743ff48409839b2564606f5e9 2024-11-27T11:36:14,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/dffe295a6f7b4e0fab1acef4837d50b9 is 50, key is test_row_0/B:col10/1732707373170/Put/seqid=0 2024-11-27T11:36:14,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742097_1273 (size=12151) 2024-11-27T11:36:14,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:14,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T11:36:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707434962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707434962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707434963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,965 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707434963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707434963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:14,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:14,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:14,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:15,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:15,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/dffe295a6f7b4e0fab1acef4837d50b9 2024-11-27T11:36:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T11:36:15,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/599d72ab808143b5bcbc9366219a1107 is 50, key is test_row_0/C:col10/1732707373170/Put/seqid=0 2024-11-27T11:36:15,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742098_1274 (size=12151) 2024-11-27T11:36:15,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:15,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707435473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:15,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707435473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:15,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707435473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707435473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:15,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707435473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:15,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:15,554 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/599d72ab808143b5bcbc9366219a1107 2024-11-27T11:36:15,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/b2dd94d743ff48409839b2564606f5e9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b2dd94d743ff48409839b2564606f5e9 2024-11-27T11:36:15,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b2dd94d743ff48409839b2564606f5e9, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T11:36:15,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/dffe295a6f7b4e0fab1acef4837d50b9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/dffe295a6f7b4e0fab1acef4837d50b9 2024-11-27T11:36:15,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/dffe295a6f7b4e0fab1acef4837d50b9, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T11:36:15,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/599d72ab808143b5bcbc9366219a1107 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/599d72ab808143b5bcbc9366219a1107 2024-11-27T11:36:15,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/599d72ab808143b5bcbc9366219a1107, entries=150, sequenceid=177, filesize=11.9 K 2024-11-27T11:36:15,572 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 5e109c5702a251266cfb057894b39f84 in 1257ms, sequenceid=177, compaction requested=true 2024-11-27T11:36:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:15,572 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:15,572 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:15,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:15,573 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:15,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:15,573 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:15,573 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:15,573 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,573 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:15,574 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:15,574 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e3539f76ccba416c8f60a412ba15e08a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/6b8684f94e1a423e8e1adc78f74159ba, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b2dd94d743ff48409839b2564606f5e9] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.9 K 2024-11-27T11:36:15,574 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,574 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/908595be150b4b878584a74e1b04a711, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a242a5e699414786a664131d7878c7af, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/dffe295a6f7b4e0fab1acef4837d50b9] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.9 K 2024-11-27T11:36:15,574 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3539f76ccba416c8f60a412ba15e08a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732707370835 2024-11-27T11:36:15,574 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 908595be150b4b878584a74e1b04a711, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732707370835 2024-11-27T11:36:15,574 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a242a5e699414786a664131d7878c7af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732707372004 2024-11-27T11:36:15,574 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b8684f94e1a423e8e1adc78f74159ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732707372004 2024-11-27T11:36:15,575 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting dffe295a6f7b4e0fab1acef4837d50b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732707373170 2024-11-27T11:36:15,575 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2dd94d743ff48409839b2564606f5e9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732707373170 2024-11-27T11:36:15,585 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:15,585 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/d059eb0bf4474795a03ea7f1ca4896cd is 50, key is test_row_0/B:col10/1732707373170/Put/seqid=0 2024-11-27T11:36:15,588 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#230 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:15,588 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/86d4ae80961040d6842ad4edc2bf5b2f is 50, key is test_row_0/A:col10/1732707373170/Put/seqid=0 2024-11-27T11:36:15,588 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-27T11:36:15,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,589 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T11:36:15,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742099_1275 (size=12561) 2024-11-27T11:36:15,597 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/d059eb0bf4474795a03ea7f1ca4896cd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d059eb0bf4474795a03ea7f1ca4896cd 2024-11-27T11:36:15,602 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into d059eb0bf4474795a03ea7f1ca4896cd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:15,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:15,602 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=13, startTime=1732707375572; duration=0sec 2024-11-27T11:36:15,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:15,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:15,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:15,603 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:15,603 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:15,603 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:15,603 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/2606bb3776d34f2aa610b843632fad5f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ee24fcc9a16f4097942957db48717dae, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/599d72ab808143b5bcbc9366219a1107] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=35.9 K 2024-11-27T11:36:15,604 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2606bb3776d34f2aa610b843632fad5f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732707370835 2024-11-27T11:36:15,604 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ee24fcc9a16f4097942957db48717dae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732707372004 2024-11-27T11:36:15,605 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 599d72ab808143b5bcbc9366219a1107, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732707373170 2024-11-27T11:36:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e1efc2dd720147db837d646e90ab4d5b is 50, key is test_row_0/A:col10/1732707374332/Put/seqid=0 2024-11-27T11:36:15,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742100_1276 (size=12561) 2024-11-27T11:36:15,622 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:15,623 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/783352c93fb444bb99dea1e5bd71798d is 50, key is test_row_0/C:col10/1732707373170/Put/seqid=0 2024-11-27T11:36:15,628 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/86d4ae80961040d6842ad4edc2bf5b2f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/86d4ae80961040d6842ad4edc2bf5b2f 2024-11-27T11:36:15,634 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into 86d4ae80961040d6842ad4edc2bf5b2f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:15,634 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:15,634 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=13, startTime=1732707375572; duration=0sec 2024-11-27T11:36:15,634 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:15,634 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:15,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742102_1278 (size=12561) 2024-11-27T11:36:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T11:36:15,657 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/783352c93fb444bb99dea1e5bd71798d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/783352c93fb444bb99dea1e5bd71798d 2024-11-27T11:36:15,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742101_1277 (size=12151) 2024-11-27T11:36:15,660 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e1efc2dd720147db837d646e90ab4d5b 2024-11-27T11:36:15,665 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into 783352c93fb444bb99dea1e5bd71798d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:15,666 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:15,666 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=13, startTime=1732707375572; duration=0sec 2024-11-27T11:36:15,666 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:15,666 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/cd853c3e485b4932bf49207911683daa is 50, key is test_row_0/B:col10/1732707374332/Put/seqid=0 2024-11-27T11:36:15,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742103_1279 (size=12151) 2024-11-27T11:36:15,682 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/cd853c3e485b4932bf49207911683daa 2024-11-27T11:36:15,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/5f0d8783c34c498ea784341ac25d262e is 50, key is test_row_0/C:col10/1732707374332/Put/seqid=0 2024-11-27T11:36:15,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742104_1280 (size=12151) 2024-11-27T11:36:16,106 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/5f0d8783c34c498ea784341ac25d262e 2024-11-27T11:36:16,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/e1efc2dd720147db837d646e90ab4d5b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e1efc2dd720147db837d646e90ab4d5b 2024-11-27T11:36:16,121 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e1efc2dd720147db837d646e90ab4d5b, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T11:36:16,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/cd853c3e485b4932bf49207911683daa as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/cd853c3e485b4932bf49207911683daa 2024-11-27T11:36:16,126 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/cd853c3e485b4932bf49207911683daa, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T11:36:16,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/5f0d8783c34c498ea784341ac25d262e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/5f0d8783c34c498ea784341ac25d262e 2024-11-27T11:36:16,151 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/5f0d8783c34c498ea784341ac25d262e, entries=150, sequenceid=193, filesize=11.9 K 2024-11-27T11:36:16,153 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 5e109c5702a251266cfb057894b39f84 in 563ms, sequenceid=193, compaction requested=false 2024-11-27T11:36:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:16,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-27T11:36:16,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-27T11:36:16,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-27T11:36:16,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6380 sec 2024-11-27T11:36:16,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.6440 sec 2024-11-27T11:36:16,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:16,489 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:16,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:36:16,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/2dd998688a644a519dda9ca75d45c6c9 is 50, key is test_row_0/A:col10/1732707376485/Put/seqid=0 2024-11-27T11:36:16,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742105_1281 (size=14541) 2024-11-27T11:36:16,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707436539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707436540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707436540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707436542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707436542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707436652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707436653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707436653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707436653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707436653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-27T11:36:16,662 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:16,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-27T11:36:16,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:16,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:16,665 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:16,666 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:16,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:16,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:16,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:16,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:16,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707436863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,865 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707436863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707436863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707436863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707436863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:16,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:16,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/2dd998688a644a519dda9ca75d45c6c9 2024-11-27T11:36:16,908 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/9e25ad816ffe47d1bdfb3445c6ec52e6 is 50, key is test_row_0/B:col10/1732707376485/Put/seqid=0 2024-11-27T11:36:16,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742106_1282 (size=12151) 2024-11-27T11:36:16,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:16,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:16,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:16,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:17,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707437173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707437173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707437173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707437173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707437173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:17,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/9e25ad816ffe47d1bdfb3445c6ec52e6 2024-11-27T11:36:17,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/770f76b252574a8c9e9025367820afb0 is 50, key is test_row_0/C:col10/1732707376485/Put/seqid=0 2024-11-27T11:36:17,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742107_1283 (size=12151) 2024-11-27T11:36:17,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707437682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707437682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707437682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707437682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707437682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:17,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/770f76b252574a8c9e9025367820afb0 2024-11-27T11:36:17,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/2dd998688a644a519dda9ca75d45c6c9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/2dd998688a644a519dda9ca75d45c6c9 2024-11-27T11:36:17,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/2dd998688a644a519dda9ca75d45c6c9, entries=200, sequenceid=208, filesize=14.2 K 2024-11-27T11:36:17,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/9e25ad816ffe47d1bdfb3445c6ec52e6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9e25ad816ffe47d1bdfb3445c6ec52e6 2024-11-27T11:36:17,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:17,742 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9e25ad816ffe47d1bdfb3445c6ec52e6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T11:36:17,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/770f76b252574a8c9e9025367820afb0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/770f76b252574a8c9e9025367820afb0 2024-11-27T11:36:17,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/770f76b252574a8c9e9025367820afb0, entries=150, sequenceid=208, filesize=11.9 K 2024-11-27T11:36:17,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 5e109c5702a251266cfb057894b39f84 in 1258ms, sequenceid=208, compaction requested=true 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:17,747 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:17,747 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:17,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:17,748 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:17,748 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:17,748 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:17,749 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,749 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:17,749 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/86d4ae80961040d6842ad4edc2bf5b2f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e1efc2dd720147db837d646e90ab4d5b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/2dd998688a644a519dda9ca75d45c6c9] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=38.3 K 2024-11-27T11:36:17,749 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,749 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d059eb0bf4474795a03ea7f1ca4896cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/cd853c3e485b4932bf49207911683daa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9e25ad816ffe47d1bdfb3445c6ec52e6] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=36.0 K 2024-11-27T11:36:17,749 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86d4ae80961040d6842ad4edc2bf5b2f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732707373170 2024-11-27T11:36:17,749 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d059eb0bf4474795a03ea7f1ca4896cd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732707373170 2024-11-27T11:36:17,749 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1efc2dd720147db837d646e90ab4d5b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732707374330 2024-11-27T11:36:17,750 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting cd853c3e485b4932bf49207911683daa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732707374330 2024-11-27T11:36:17,750 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dd998688a644a519dda9ca75d45c6c9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732707376485 2024-11-27T11:36:17,750 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e25ad816ffe47d1bdfb3445c6ec52e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732707376485 2024-11-27T11:36:17,758 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:17,759 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/fe4d9be7e20648179cdc7dbfefbc7e0c is 50, key is test_row_0/B:col10/1732707376485/Put/seqid=0 2024-11-27T11:36:17,766 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:17,767 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/5e4b97bc83c04f4d8c89510cdd986416 is 50, key is test_row_0/A:col10/1732707376485/Put/seqid=0 2024-11-27T11:36:17,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742108_1284 (size=12663) 2024-11-27T11:36:17,773 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/fe4d9be7e20648179cdc7dbfefbc7e0c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/fe4d9be7e20648179cdc7dbfefbc7e0c 2024-11-27T11:36:17,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742109_1285 (size=12663) 2024-11-27T11:36:17,779 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into fe4d9be7e20648179cdc7dbfefbc7e0c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:17,779 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:17,779 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=13, startTime=1732707377747; duration=0sec 2024-11-27T11:36:17,779 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:17,779 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:17,779 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:17,780 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:17,780 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:17,780 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,780 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/783352c93fb444bb99dea1e5bd71798d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/5f0d8783c34c498ea784341ac25d262e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/770f76b252574a8c9e9025367820afb0] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=36.0 K 2024-11-27T11:36:17,781 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 783352c93fb444bb99dea1e5bd71798d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732707373170 2024-11-27T11:36:17,781 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f0d8783c34c498ea784341ac25d262e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732707374330 2024-11-27T11:36:17,782 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 770f76b252574a8c9e9025367820afb0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732707376485 2024-11-27T11:36:17,788 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:17,789 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/ddbae050f26d43ef990fad57685a1501 is 50, key is test_row_0/C:col10/1732707376485/Put/seqid=0 2024-11-27T11:36:17,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742110_1286 (size=12663) 2024-11-27T11:36:17,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:17,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-27T11:36:17,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:17,890 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:36:17,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/d093ab77e0424f6485b2840814c0b76b is 50, key is test_row_0/A:col10/1732707376540/Put/seqid=0 2024-11-27T11:36:17,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742111_1287 (size=12151) 2024-11-27T11:36:18,181 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/5e4b97bc83c04f4d8c89510cdd986416 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/5e4b97bc83c04f4d8c89510cdd986416 2024-11-27T11:36:18,186 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into 5e4b97bc83c04f4d8c89510cdd986416(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:18,186 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:18,186 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=13, startTime=1732707377747; duration=0sec 2024-11-27T11:36:18,186 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:18,186 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:18,200 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/ddbae050f26d43ef990fad57685a1501 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ddbae050f26d43ef990fad57685a1501 2024-11-27T11:36:18,205 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into ddbae050f26d43ef990fad57685a1501(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:18,205 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:18,205 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=13, startTime=1732707377747; duration=0sec 2024-11-27T11:36:18,205 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:18,205 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:18,300 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/d093ab77e0424f6485b2840814c0b76b 2024-11-27T11:36:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/9c69839e17404b93aee27e738c582989 is 50, key is test_row_0/B:col10/1732707376540/Put/seqid=0 2024-11-27T11:36:18,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742112_1288 (size=12151) 2024-11-27T11:36:18,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:18,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707438701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707438701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707438702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707438702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707438704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,707 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in 5e109c5702a251266cfb057894b39f84 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,712 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/9c69839e17404b93aee27e738c582989 2024-11-27T11:36:18,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/d086fa513ea0485a8d2e85e9a59183fc is 50, key is test_row_0/C:col10/1732707376540/Put/seqid=0 2024-11-27T11:36:18,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742113_1289 (size=12151) 2024-11-27T11:36:18,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:18,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707438812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707438812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,815 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707438812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707438813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:18,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707439022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707439022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707439022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707439022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,125 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/d086fa513ea0485a8d2e85e9a59183fc 2024-11-27T11:36:19,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/d093ab77e0424f6485b2840814c0b76b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d093ab77e0424f6485b2840814c0b76b 2024-11-27T11:36:19,134 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d093ab77e0424f6485b2840814c0b76b, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T11:36:19,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/9c69839e17404b93aee27e738c582989 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9c69839e17404b93aee27e738c582989 2024-11-27T11:36:19,138 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9c69839e17404b93aee27e738c582989, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T11:36:19,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/d086fa513ea0485a8d2e85e9a59183fc as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d086fa513ea0485a8d2e85e9a59183fc 2024-11-27T11:36:19,143 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d086fa513ea0485a8d2e85e9a59183fc, entries=150, sequenceid=233, filesize=11.9 K 2024-11-27T11:36:19,144 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 5e109c5702a251266cfb057894b39f84 in 1254ms, sequenceid=233, compaction requested=false 2024-11-27T11:36:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:19,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-27T11:36:19,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-27T11:36:19,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-27T11:36:19,148 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4810 sec 2024-11-27T11:36:19,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.4860 sec 2024-11-27T11:36:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:19,336 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:19,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T11:36:19,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/c7fe25204c034b42871863f59b6dca53 is 50, key is test_row_0/A:col10/1732707379335/Put/seqid=0 2024-11-27T11:36:19,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742114_1290 (size=14541) 2024-11-27T11:36:19,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/c7fe25204c034b42871863f59b6dca53 2024-11-27T11:36:19,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/c6cf075554184bff87539da1cdb62555 is 50, key is test_row_0/B:col10/1732707379335/Put/seqid=0 2024-11-27T11:36:19,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742115_1291 (size=12151) 2024-11-27T11:36:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707439365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707439365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707439366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707439368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,372 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707439472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707439472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707439482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707439482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707439683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707439683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707439693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707439693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,695 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/c6cf075554184bff87539da1cdb62555 2024-11-27T11:36:19,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6c753f0ae8f94535b50c7182e3201344 is 50, key is test_row_0/C:col10/1732707379335/Put/seqid=0 2024-11-27T11:36:19,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742116_1292 (size=12151) 2024-11-27T11:36:19,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707439992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:19,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707439992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:19,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707440002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,006 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707440003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6c753f0ae8f94535b50c7182e3201344 2024-11-27T11:36:20,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/c7fe25204c034b42871863f59b6dca53 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/c7fe25204c034b42871863f59b6dca53 2024-11-27T11:36:20,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/c7fe25204c034b42871863f59b6dca53, entries=200, sequenceid=251, filesize=14.2 K 2024-11-27T11:36:20,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/c6cf075554184bff87539da1cdb62555 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/c6cf075554184bff87539da1cdb62555 2024-11-27T11:36:20,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/c6cf075554184bff87539da1cdb62555, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T11:36:20,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6c753f0ae8f94535b50c7182e3201344 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6c753f0ae8f94535b50c7182e3201344 2024-11-27T11:36:20,189 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6c753f0ae8f94535b50c7182e3201344, entries=150, sequenceid=251, filesize=11.9 K 2024-11-27T11:36:20,189 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5e109c5702a251266cfb057894b39f84 in 853ms, sequenceid=251, compaction requested=true 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:20,190 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:20,190 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:20,190 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:20,192 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:20,192 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:20,192 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:20,192 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:20,192 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:20,192 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:20,192 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/5e4b97bc83c04f4d8c89510cdd986416, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d093ab77e0424f6485b2840814c0b76b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/c7fe25204c034b42871863f59b6dca53] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=38.4 K 2024-11-27T11:36:20,192 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/fe4d9be7e20648179cdc7dbfefbc7e0c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9c69839e17404b93aee27e738c582989, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/c6cf075554184bff87539da1cdb62555] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=36.1 K 2024-11-27T11:36:20,192 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e4b97bc83c04f4d8c89510cdd986416, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732707376485 2024-11-27T11:36:20,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting fe4d9be7e20648179cdc7dbfefbc7e0c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732707376485 2024-11-27T11:36:20,193 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d093ab77e0424f6485b2840814c0b76b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732707376539 2024-11-27T11:36:20,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c69839e17404b93aee27e738c582989, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732707376539 2024-11-27T11:36:20,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c6cf075554184bff87539da1cdb62555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732707379335 2024-11-27T11:36:20,194 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7fe25204c034b42871863f59b6dca53, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732707378700 2024-11-27T11:36:20,201 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:20,201 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:20,202 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/2290cfde18a14e9790da11ce33540274 is 50, key is test_row_0/B:col10/1732707379335/Put/seqid=0 2024-11-27T11:36:20,202 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/217e70985332496d9bbbf93432e829e6 is 50, key is test_row_0/A:col10/1732707379335/Put/seqid=0 2024-11-27T11:36:20,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742118_1294 (size=12765) 2024-11-27T11:36:20,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742117_1293 (size=12765) 2024-11-27T11:36:20,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:20,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:20,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T11:36:20,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/7be89dbc44124e7a82442f233e393edc is 50, key is test_row_0/A:col10/1732707379364/Put/seqid=0 2024-11-27T11:36:20,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742119_1295 (size=12301) 2024-11-27T11:36:20,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707440524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,527 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707440525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707440526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707440526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,612 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/2290cfde18a14e9790da11ce33540274 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/2290cfde18a14e9790da11ce33540274 2024-11-27T11:36:20,612 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/217e70985332496d9bbbf93432e829e6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/217e70985332496d9bbbf93432e829e6 2024-11-27T11:36:20,618 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into 2290cfde18a14e9790da11ce33540274(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:20,618 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into 217e70985332496d9bbbf93432e829e6(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:20,618 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=13, startTime=1732707380190; duration=0sec 2024-11-27T11:36:20,618 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=13, startTime=1732707380190; duration=0sec 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:20,618 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:20,619 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:20,619 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:20,619 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:20,619 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ddbae050f26d43ef990fad57685a1501, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d086fa513ea0485a8d2e85e9a59183fc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6c753f0ae8f94535b50c7182e3201344] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=36.1 K 2024-11-27T11:36:20,620 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ddbae050f26d43ef990fad57685a1501, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732707376485 2024-11-27T11:36:20,620 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d086fa513ea0485a8d2e85e9a59183fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732707376539 2024-11-27T11:36:20,620 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c753f0ae8f94535b50c7182e3201344, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732707379335 2024-11-27T11:36:20,626 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:20,627 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6d479dc920ae48b783443ccc7b525214 is 50, key is test_row_0/C:col10/1732707379335/Put/seqid=0 2024-11-27T11:36:20,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742120_1296 (size=12765) 2024-11-27T11:36:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707440633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707440633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707440642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,644 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707440642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57348 deadline: 1732707440723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,727 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in 5e109c5702a251266cfb057894b39f84 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-27T11:36:20,833 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:20,833 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-27T11:36:20,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:20,835 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:20,836 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:20,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:20,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707440843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707440843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,847 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707440852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707440852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:20,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:20,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/7be89dbc44124e7a82442f233e393edc 2024-11-27T11:36:20,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/a1f0560b5f8c4afc8f71118d511c289d is 50, key is test_row_0/B:col10/1732707379364/Put/seqid=0 2024-11-27T11:36:20,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742121_1297 (size=12301) 2024-11-27T11:36:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:20,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:20,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:20,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:20,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:20,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,036 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6d479dc920ae48b783443ccc7b525214 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6d479dc920ae48b783443ccc7b525214 2024-11-27T11:36:21,040 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into 6d479dc920ae48b783443ccc7b525214(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:21,040 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:21,040 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=13, startTime=1732707380190; duration=0sec 2024-11-27T11:36:21,041 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:21,041 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:21,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707441153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707441153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,155 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707441162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,164 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707441162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,165 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,294 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/a1f0560b5f8c4afc8f71118d511c289d 2024-11-27T11:36:21,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/87b1123acc1a4e34be3184961a81e99e is 50, key is test_row_0/C:col10/1732707379364/Put/seqid=0 2024-11-27T11:36:21,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742122_1298 (size=12301) 2024-11-27T11:36:21,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:21,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707441663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707441663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707441672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707441672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:21,676 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:21,741 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/87b1123acc1a4e34be3184961a81e99e 2024-11-27T11:36:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/7be89dbc44124e7a82442f233e393edc as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/7be89dbc44124e7a82442f233e393edc 2024-11-27T11:36:21,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/7be89dbc44124e7a82442f233e393edc, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T11:36:21,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/a1f0560b5f8c4afc8f71118d511c289d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a1f0560b5f8c4afc8f71118d511c289d 2024-11-27T11:36:21,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a1f0560b5f8c4afc8f71118d511c289d, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T11:36:21,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:21,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/87b1123acc1a4e34be3184961a81e99e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/87b1123acc1a4e34be3184961a81e99e 2024-11-27T11:36:21,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/87b1123acc1a4e34be3184961a81e99e, entries=150, sequenceid=272, filesize=12.0 K 2024-11-27T11:36:21,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 5e109c5702a251266cfb057894b39f84 in 1255ms, sequenceid=272, compaction requested=false 2024-11-27T11:36:21,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:21,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-27T11:36:21,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:21,907 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T11:36:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/3ef889f2ae2b42718a2b25607d287446 is 50, key is test_row_0/A:col10/1732707380522/Put/seqid=0 2024-11-27T11:36:21,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742123_1299 (size=12301) 2024-11-27T11:36:21,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:22,317 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/3ef889f2ae2b42718a2b25607d287446 2024-11-27T11:36:22,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/26bb1289f6264f1199d9043c469ada2d is 50, key is test_row_0/B:col10/1732707380522/Put/seqid=0 2024-11-27T11:36:22,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742124_1300 (size=12301) 2024-11-27T11:36:22,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:22,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. as already flushing 2024-11-27T11:36:22,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707442705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707442708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707442709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707442710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,730 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/26bb1289f6264f1199d9043c469ada2d 2024-11-27T11:36:22,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/f67623e76f794ed0aa0681c46f8e3fef is 50, key is test_row_0/C:col10/1732707380522/Put/seqid=0 2024-11-27T11:36:22,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742125_1301 (size=12301) 2024-11-27T11:36:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707442822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707442822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707442822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707442823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:22,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:23,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707443032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707443032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707443032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707443033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,154 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/f67623e76f794ed0aa0681c46f8e3fef 2024-11-27T11:36:23,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/3ef889f2ae2b42718a2b25607d287446 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/3ef889f2ae2b42718a2b25607d287446 2024-11-27T11:36:23,162 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/3ef889f2ae2b42718a2b25607d287446, entries=150, sequenceid=291, filesize=12.0 K 2024-11-27T11:36:23,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/26bb1289f6264f1199d9043c469ada2d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/26bb1289f6264f1199d9043c469ada2d 2024-11-27T11:36:23,166 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/26bb1289f6264f1199d9043c469ada2d, entries=150, sequenceid=291, filesize=12.0 K 2024-11-27T11:36:23,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/f67623e76f794ed0aa0681c46f8e3fef as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f67623e76f794ed0aa0681c46f8e3fef 2024-11-27T11:36:23,171 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f67623e76f794ed0aa0681c46f8e3fef, entries=150, sequenceid=291, filesize=12.0 K 2024-11-27T11:36:23,172 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 5e109c5702a251266cfb057894b39f84 in 1264ms, sequenceid=291, compaction requested=true 2024-11-27T11:36:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:23,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-27T11:36:23,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-27T11:36:23,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-27T11:36:23,175 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3370 sec 2024-11-27T11:36:23,177 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.3420 sec 2024-11-27T11:36:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:23,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:23,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-27T11:36:23,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c is 50, key is test_row_0/A:col10/1732707383346/Put/seqid=0 2024-11-27T11:36:23,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742126_1302 (size=17181) 2024-11-27T11:36:23,357 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c 2024-11-27T11:36:23,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/6af570f188c9476eb45cbd0a4c937df7 is 50, key is test_row_0/B:col10/1732707383346/Put/seqid=0 2024-11-27T11:36:23,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742127_1303 (size=12301) 2024-11-27T11:36:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707443390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707443389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707443391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707443392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707443502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707443503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707443503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,506 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707443503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707443712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707443713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:23,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:23,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/6af570f188c9476eb45cbd0a4c937df7 2024-11-27T11:36:23,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2 is 50, key is test_row_0/C:col10/1732707383346/Put/seqid=0 2024-11-27T11:36:23,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742128_1304 (size=12301) 2024-11-27T11:36:23,905 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-27T11:36:23,905 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:23,905 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,906 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,906 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:23,907 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-27T11:36:23,907 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:23,907 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,907 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:23,907 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-27T11:36:23,907 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:23,907 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,907 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:23,909 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-27T11:36:23,909 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:23,909 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,909 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:23,910 INFO [Thread-1120 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1120. 2024-11-27T11:36:23,910 DEBUG [Thread-1120 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:23,910 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,910 DEBUG [Thread-1120 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:23,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57266 deadline: 1732707444022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57346 deadline: 1732707444022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:24,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57308 deadline: 1732707444022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:24,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:57356 deadline: 1732707444022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:24,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:36:24,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2 2024-11-27T11:36:24,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c 2024-11-27T11:36:24,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c, entries=250, sequenceid=315, filesize=16.8 K 2024-11-27T11:36:24,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/6af570f188c9476eb45cbd0a4c937df7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/6af570f188c9476eb45cbd0a4c937df7 2024-11-27T11:36:24,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/6af570f188c9476eb45cbd0a4c937df7, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T11:36:24,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2 2024-11-27T11:36:24,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2, entries=150, sequenceid=315, filesize=12.0 K 2024-11-27T11:36:24,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 5e109c5702a251266cfb057894b39f84 in 846ms, sequenceid=315, compaction requested=true 2024-11-27T11:36:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:24,192 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:36:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:24,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:24,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5e109c5702a251266cfb057894b39f84:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:24,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:36:24,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:24,193 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:36:24,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:36:24,193 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/A is initiating minor compaction (all files) 2024-11-27T11:36:24,193 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/B is initiating minor compaction (all files) 2024-11-27T11:36:24,194 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/A in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:24,194 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/B in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:24,194 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/217e70985332496d9bbbf93432e829e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/7be89dbc44124e7a82442f233e393edc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/3ef889f2ae2b42718a2b25607d287446, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=53.3 K 2024-11-27T11:36:24,194 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/2290cfde18a14e9790da11ce33540274, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a1f0560b5f8c4afc8f71118d511c289d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/26bb1289f6264f1199d9043c469ada2d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/6af570f188c9476eb45cbd0a4c937df7] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=48.5 K 2024-11-27T11:36:24,194 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 217e70985332496d9bbbf93432e829e6, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732707379335 2024-11-27T11:36:24,194 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2290cfde18a14e9790da11ce33540274, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732707379335 2024-11-27T11:36:24,194 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7be89dbc44124e7a82442f233e393edc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732707379356 2024-11-27T11:36:24,194 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f0560b5f8c4afc8f71118d511c289d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732707379356 2024-11-27T11:36:24,194 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ef889f2ae2b42718a2b25607d287446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732707380519 2024-11-27T11:36:24,195 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 26bb1289f6264f1199d9043c469ada2d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732707380519 2024-11-27T11:36:24,195 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcbb1c6e60cc4bdcbfdedd059e8dfc7c, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732707382709 2024-11-27T11:36:24,195 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6af570f188c9476eb45cbd0a4c937df7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732707383346 2024-11-27T11:36:24,203 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:24,204 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/7fd82d70a4f240818207fb0f3c040cef is 50, key is test_row_0/B:col10/1732707383346/Put/seqid=0 2024-11-27T11:36:24,208 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#A#compaction#260 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:24,209 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/29e260c325104c3481fe312e55a705ae is 50, key is test_row_0/A:col10/1732707383346/Put/seqid=0 2024-11-27T11:36:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742129_1305 (size=13051) 2024-11-27T11:36:24,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742130_1306 (size=13051) 2024-11-27T11:36:24,216 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/29e260c325104c3481fe312e55a705ae as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/29e260c325104c3481fe312e55a705ae 2024-11-27T11:36:24,220 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e109c5702a251266cfb057894b39f84/A of 5e109c5702a251266cfb057894b39f84 into 29e260c325104c3481fe312e55a705ae(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:24,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:24,220 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/A, priority=12, startTime=1732707384192; duration=0sec 2024-11-27T11:36:24,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:24,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:A 2024-11-27T11:36:24,220 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:36:24,221 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49668 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:36:24,221 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 5e109c5702a251266cfb057894b39f84/C is initiating minor compaction (all files) 2024-11-27T11:36:24,221 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5e109c5702a251266cfb057894b39f84/C in TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:24,222 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6d479dc920ae48b783443ccc7b525214, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/87b1123acc1a4e34be3184961a81e99e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f67623e76f794ed0aa0681c46f8e3fef, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp, totalSize=48.5 K 2024-11-27T11:36:24,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d479dc920ae48b783443ccc7b525214, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732707379335 2024-11-27T11:36:24,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87b1123acc1a4e34be3184961a81e99e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1732707379356 2024-11-27T11:36:24,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f67623e76f794ed0aa0681c46f8e3fef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732707380519 2024-11-27T11:36:24,223 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b0a2a3caaf349e9bd2e458d7ea0f3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732707383346 2024-11-27T11:36:24,230 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5e109c5702a251266cfb057894b39f84#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:24,230 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/f071ded02b454c8fbd1669ab9012542e is 50, key is test_row_0/C:col10/1732707383346/Put/seqid=0 2024-11-27T11:36:24,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742131_1307 (size=13051) 2024-11-27T11:36:24,237 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/f071ded02b454c8fbd1669ab9012542e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f071ded02b454c8fbd1669ab9012542e 2024-11-27T11:36:24,241 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e109c5702a251266cfb057894b39f84/C of 5e109c5702a251266cfb057894b39f84 into f071ded02b454c8fbd1669ab9012542e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:24,241 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:24,241 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/C, priority=12, startTime=1732707384192; duration=0sec 2024-11-27T11:36:24,242 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:24,242 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:C 2024-11-27T11:36:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:24,534 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:24,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-27T11:36:24,535 INFO [Thread-1111 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1111. 2024-11-27T11:36:24,535 INFO [Thread-1109 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1109. 2024-11-27T11:36:24,535 INFO [Thread-1115 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1115. 2024-11-27T11:36:24,535 DEBUG [Thread-1109 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:24,535 DEBUG [Thread-1111 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:24,535 DEBUG [Thread-1115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:24,535 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,535 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,535 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,535 DEBUG [Thread-1115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,535 DEBUG [Thread-1109 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,535 DEBUG [Thread-1111 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,535 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,536 INFO [Thread-1117 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1117. 2024-11-27T11:36:24,536 DEBUG [Thread-1117 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:24,536 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,536 DEBUG [Thread-1117 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/58d7f5fb8e254a938deb86cd0e2033ec is 50, key is test_row_0/A:col10/1732707383389/Put/seqid=0 2024-11-27T11:36:24,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742132_1308 (size=12301) 2024-11-27T11:36:24,614 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/7fd82d70a4f240818207fb0f3c040cef as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/7fd82d70a4f240818207fb0f3c040cef 2024-11-27T11:36:24,619 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5e109c5702a251266cfb057894b39f84/B of 5e109c5702a251266cfb057894b39f84 into 7fd82d70a4f240818207fb0f3c040cef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:24,619 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:24,619 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., storeName=5e109c5702a251266cfb057894b39f84/B, priority=12, startTime=1732707384192; duration=0sec 2024-11-27T11:36:24,619 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:24,619 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5e109c5702a251266cfb057894b39f84:B 2024-11-27T11:36:24,754 INFO [Thread-1113 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1113. 2024-11-27T11:36:24,754 DEBUG [Thread-1113 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:24,754 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,754 DEBUG [Thread-1113 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,944 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/58d7f5fb8e254a938deb86cd0e2033ec 2024-11-27T11:36:24,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/b57d01b4d8744a9e951b8f7c3bd981aa is 50, key is test_row_0/B:col10/1732707383389/Put/seqid=0 2024-11-27T11:36:24,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742133_1309 (size=12301) 2024-11-27T11:36:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-27T11:36:24,992 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3738 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3658 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3665 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3725 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3665 2024-11-27T11:36:24,993 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T11:36:24,993 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:36:24,993 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:36:24,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:24,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:24,994 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T11:36:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T11:36:24,997 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707384996"}]},"ts":"1732707384996"} 2024-11-27T11:36:24,998 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T11:36:24,998 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T11:36:24,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:36:25,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, UNASSIGN}] 2024-11-27T11:36:25,001 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, UNASSIGN 2024-11-27T11:36:25,001 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5e109c5702a251266cfb057894b39f84, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:25,002 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, UNASSIGN because future has completed 2024-11-27T11:36:25,002 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:36:25,002 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:36:25,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T11:36:25,154 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:25,154 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:36:25,155 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing 5e109c5702a251266cfb057894b39f84, disabling compactions & flushes 2024-11-27T11:36:25,155 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:25,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T11:36:25,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/b57d01b4d8744a9e951b8f7c3bd981aa 2024-11-27T11:36:25,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/e1d85dfaf5d2488a92b2f5c31546c550 is 50, key is test_row_0/C:col10/1732707383389/Put/seqid=0 2024-11-27T11:36:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742134_1310 (size=12301) 2024-11-27T11:36:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T11:36:25,763 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/e1d85dfaf5d2488a92b2f5c31546c550 2024-11-27T11:36:25,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/58d7f5fb8e254a938deb86cd0e2033ec as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/58d7f5fb8e254a938deb86cd0e2033ec 2024-11-27T11:36:25,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/58d7f5fb8e254a938deb86cd0e2033ec, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T11:36:25,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/b57d01b4d8744a9e951b8f7c3bd981aa as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b57d01b4d8744a9e951b8f7c3bd981aa 2024-11-27T11:36:25,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b57d01b4d8744a9e951b8f7c3bd981aa, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T11:36:25,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/e1d85dfaf5d2488a92b2f5c31546c550 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/e1d85dfaf5d2488a92b2f5c31546c550 2024-11-27T11:36:25,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/e1d85dfaf5d2488a92b2f5c31546c550, entries=150, sequenceid=334, filesize=12.0 K 2024-11-27T11:36:25,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=6.71 KB/6870 for 5e109c5702a251266cfb057894b39f84 in 1244ms, sequenceid=334, compaction requested=false 2024-11-27T11:36:25,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5e109c5702a251266cfb057894b39f84: 2024-11-27T11:36:25,779 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:25,779 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:25,779 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. after waiting 0 ms 2024-11-27T11:36:25,779 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:25,779 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing 5e109c5702a251266cfb057894b39f84 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-27T11:36:25,782 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/d5140582b32d47a38af46021461a8ba2 is 50, key is test_row_0/A:col10/1732707384753/Put/seqid=0 2024-11-27T11:36:25,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742135_1311 (size=7415) 2024-11-27T11:36:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T11:36:26,186 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/d5140582b32d47a38af46021461a8ba2 2024-11-27T11:36:26,192 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/45caed7b8d6649b08d2ccc87ee03af99 is 50, key is test_row_0/B:col10/1732707384753/Put/seqid=0 2024-11-27T11:36:26,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742136_1312 (size=7415) 2024-11-27T11:36:26,596 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/45caed7b8d6649b08d2ccc87ee03af99 2024-11-27T11:36:26,601 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/563d6ac52e0f43648c526b9972b26dbf is 50, key is test_row_0/C:col10/1732707384753/Put/seqid=0 2024-11-27T11:36:26,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742137_1313 (size=7415) 2024-11-27T11:36:27,005 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=339 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/563d6ac52e0f43648c526b9972b26dbf 2024-11-27T11:36:27,009 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/A/d5140582b32d47a38af46021461a8ba2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d5140582b32d47a38af46021461a8ba2 2024-11-27T11:36:27,012 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d5140582b32d47a38af46021461a8ba2, entries=50, sequenceid=339, filesize=7.2 K 2024-11-27T11:36:27,013 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/B/45caed7b8d6649b08d2ccc87ee03af99 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/45caed7b8d6649b08d2ccc87ee03af99 2024-11-27T11:36:27,016 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/45caed7b8d6649b08d2ccc87ee03af99, entries=50, sequenceid=339, filesize=7.2 K 2024-11-27T11:36:27,017 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/.tmp/C/563d6ac52e0f43648c526b9972b26dbf as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/563d6ac52e0f43648c526b9972b26dbf 2024-11-27T11:36:27,020 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/563d6ac52e0f43648c526b9972b26dbf, entries=50, sequenceid=339, filesize=7.2 K 2024-11-27T11:36:27,021 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5e109c5702a251266cfb057894b39f84 in 1242ms, sequenceid=339, compaction requested=true 2024-11-27T11:36:27,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/0ff8ae235392406eb45186550da97fa6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e6bf16a33c9c4c92955f18025102c857, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9d468fb344cb4009b3ddb9a060f6ec5c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/ff29028f13484b9f8ca792ed78b9a89f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9dcf4a7709be4300b2c3d9fac05dabd6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/60bc9e27c0aa4803a33beee3a346868a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e034db997fdd429cbe452f05dc02cc6c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/67fd6509ad9b46198cedf50432c1a7b4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e3539f76ccba416c8f60a412ba15e08a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b0f5bdb314ac4ccc913f3227ebfee595, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/6b8684f94e1a423e8e1adc78f74159ba, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/86d4ae80961040d6842ad4edc2bf5b2f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b2dd94d743ff48409839b2564606f5e9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e1efc2dd720147db837d646e90ab4d5b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/2dd998688a644a519dda9ca75d45c6c9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/5e4b97bc83c04f4d8c89510cdd986416, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d093ab77e0424f6485b2840814c0b76b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/c7fe25204c034b42871863f59b6dca53, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/217e70985332496d9bbbf93432e829e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/7be89dbc44124e7a82442f233e393edc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/3ef889f2ae2b42718a2b25607d287446, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c] to archive 2024-11-27T11:36:27,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:36:27,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/0ff8ae235392406eb45186550da97fa6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/0ff8ae235392406eb45186550da97fa6 2024-11-27T11:36:27,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e6bf16a33c9c4c92955f18025102c857 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e6bf16a33c9c4c92955f18025102c857 2024-11-27T11:36:27,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9d468fb344cb4009b3ddb9a060f6ec5c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9d468fb344cb4009b3ddb9a060f6ec5c 2024-11-27T11:36:27,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/ff29028f13484b9f8ca792ed78b9a89f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/ff29028f13484b9f8ca792ed78b9a89f 2024-11-27T11:36:27,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9dcf4a7709be4300b2c3d9fac05dabd6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/9dcf4a7709be4300b2c3d9fac05dabd6 2024-11-27T11:36:27,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/60bc9e27c0aa4803a33beee3a346868a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/60bc9e27c0aa4803a33beee3a346868a 2024-11-27T11:36:27,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e034db997fdd429cbe452f05dc02cc6c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e034db997fdd429cbe452f05dc02cc6c 2024-11-27T11:36:27,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/67fd6509ad9b46198cedf50432c1a7b4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/67fd6509ad9b46198cedf50432c1a7b4 2024-11-27T11:36:27,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e3539f76ccba416c8f60a412ba15e08a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e3539f76ccba416c8f60a412ba15e08a 2024-11-27T11:36:27,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b0f5bdb314ac4ccc913f3227ebfee595 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b0f5bdb314ac4ccc913f3227ebfee595 2024-11-27T11:36:27,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/6b8684f94e1a423e8e1adc78f74159ba to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/6b8684f94e1a423e8e1adc78f74159ba 2024-11-27T11:36:27,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/86d4ae80961040d6842ad4edc2bf5b2f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/86d4ae80961040d6842ad4edc2bf5b2f 2024-11-27T11:36:27,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b2dd94d743ff48409839b2564606f5e9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/b2dd94d743ff48409839b2564606f5e9 2024-11-27T11:36:27,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e1efc2dd720147db837d646e90ab4d5b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/e1efc2dd720147db837d646e90ab4d5b 2024-11-27T11:36:27,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/2dd998688a644a519dda9ca75d45c6c9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/2dd998688a644a519dda9ca75d45c6c9 2024-11-27T11:36:27,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/5e4b97bc83c04f4d8c89510cdd986416 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/5e4b97bc83c04f4d8c89510cdd986416 2024-11-27T11:36:27,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d093ab77e0424f6485b2840814c0b76b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d093ab77e0424f6485b2840814c0b76b 2024-11-27T11:36:27,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/c7fe25204c034b42871863f59b6dca53 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/c7fe25204c034b42871863f59b6dca53 2024-11-27T11:36:27,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/217e70985332496d9bbbf93432e829e6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/217e70985332496d9bbbf93432e829e6 2024-11-27T11:36:27,041 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/7be89dbc44124e7a82442f233e393edc to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/7be89dbc44124e7a82442f233e393edc 2024-11-27T11:36:27,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/3ef889f2ae2b42718a2b25607d287446 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/3ef889f2ae2b42718a2b25607d287446 2024-11-27T11:36:27,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/fcbb1c6e60cc4bdcbfdedd059e8dfc7c 2024-11-27T11:36:27,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/5c10a26b5162464988a8301c252cff4c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/deadf7b2bac448a8b44e13562dc55f4a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/81f6c986bf9843879f4a76317171ff19, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d26c7e5d0ed3496d82913e3345fc041d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/4c775b23cee643c4b5996ede370b1148, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/f2937869f381488ba8441c3fc105612f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/17b3c1cac242496c906150fd8f39afb1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/35f5f05992ed4803b8f5a8addebd5c5f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/908595be150b4b878584a74e1b04a711, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b304ac20701049f29d1625250ca19f57, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a242a5e699414786a664131d7878c7af, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d059eb0bf4474795a03ea7f1ca4896cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/dffe295a6f7b4e0fab1acef4837d50b9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/cd853c3e485b4932bf49207911683daa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/fe4d9be7e20648179cdc7dbfefbc7e0c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9e25ad816ffe47d1bdfb3445c6ec52e6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9c69839e17404b93aee27e738c582989, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/2290cfde18a14e9790da11ce33540274, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/c6cf075554184bff87539da1cdb62555, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a1f0560b5f8c4afc8f71118d511c289d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/26bb1289f6264f1199d9043c469ada2d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/6af570f188c9476eb45cbd0a4c937df7] to archive 2024-11-27T11:36:27,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:36:27,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/5c10a26b5162464988a8301c252cff4c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/5c10a26b5162464988a8301c252cff4c 2024-11-27T11:36:27,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/deadf7b2bac448a8b44e13562dc55f4a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/deadf7b2bac448a8b44e13562dc55f4a 2024-11-27T11:36:27,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/81f6c986bf9843879f4a76317171ff19 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/81f6c986bf9843879f4a76317171ff19 2024-11-27T11:36:27,049 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d26c7e5d0ed3496d82913e3345fc041d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d26c7e5d0ed3496d82913e3345fc041d 2024-11-27T11:36:27,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/4c775b23cee643c4b5996ede370b1148 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/4c775b23cee643c4b5996ede370b1148 2024-11-27T11:36:27,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/f2937869f381488ba8441c3fc105612f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/f2937869f381488ba8441c3fc105612f 2024-11-27T11:36:27,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/17b3c1cac242496c906150fd8f39afb1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/17b3c1cac242496c906150fd8f39afb1 2024-11-27T11:36:27,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/35f5f05992ed4803b8f5a8addebd5c5f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/35f5f05992ed4803b8f5a8addebd5c5f 2024-11-27T11:36:27,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/908595be150b4b878584a74e1b04a711 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/908595be150b4b878584a74e1b04a711 2024-11-27T11:36:27,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b304ac20701049f29d1625250ca19f57 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b304ac20701049f29d1625250ca19f57 2024-11-27T11:36:27,055 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a242a5e699414786a664131d7878c7af to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a242a5e699414786a664131d7878c7af 2024-11-27T11:36:27,056 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d059eb0bf4474795a03ea7f1ca4896cd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/d059eb0bf4474795a03ea7f1ca4896cd 2024-11-27T11:36:27,057 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/dffe295a6f7b4e0fab1acef4837d50b9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/dffe295a6f7b4e0fab1acef4837d50b9 2024-11-27T11:36:27,058 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/cd853c3e485b4932bf49207911683daa to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/cd853c3e485b4932bf49207911683daa 2024-11-27T11:36:27,059 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/fe4d9be7e20648179cdc7dbfefbc7e0c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/fe4d9be7e20648179cdc7dbfefbc7e0c 2024-11-27T11:36:27,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9e25ad816ffe47d1bdfb3445c6ec52e6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9e25ad816ffe47d1bdfb3445c6ec52e6 2024-11-27T11:36:27,060 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9c69839e17404b93aee27e738c582989 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/9c69839e17404b93aee27e738c582989 2024-11-27T11:36:27,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/2290cfde18a14e9790da11ce33540274 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/2290cfde18a14e9790da11ce33540274 2024-11-27T11:36:27,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/c6cf075554184bff87539da1cdb62555 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/c6cf075554184bff87539da1cdb62555 2024-11-27T11:36:27,063 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a1f0560b5f8c4afc8f71118d511c289d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/a1f0560b5f8c4afc8f71118d511c289d 2024-11-27T11:36:27,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/26bb1289f6264f1199d9043c469ada2d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/26bb1289f6264f1199d9043c469ada2d 2024-11-27T11:36:27,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/6af570f188c9476eb45cbd0a4c937df7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/6af570f188c9476eb45cbd0a4c937df7 2024-11-27T11:36:27,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/08752b1a91054eb98e3897fe437f8dc0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/b738e10da49b4e15bb1452e99727ecf0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/3851716dc0ae47828b5d81d6419b4b7a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/60b967beec3049559485a47594339ee7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7e7fc3cc1afc412b8d98b6022daa01c4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7267326b54fd4662b40f66751f42f0b2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/c32c36e4377046a281e3efbd08cccdd2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d418e822a22c4e0190b19c2987901cfc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/2606bb3776d34f2aa610b843632fad5f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/9e7bcc46612842ecb6f31d0b2bee5bf4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ee24fcc9a16f4097942957db48717dae, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/783352c93fb444bb99dea1e5bd71798d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/599d72ab808143b5bcbc9366219a1107, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/5f0d8783c34c498ea784341ac25d262e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ddbae050f26d43ef990fad57685a1501, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/770f76b252574a8c9e9025367820afb0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d086fa513ea0485a8d2e85e9a59183fc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6d479dc920ae48b783443ccc7b525214, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6c753f0ae8f94535b50c7182e3201344, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/87b1123acc1a4e34be3184961a81e99e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f67623e76f794ed0aa0681c46f8e3fef, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2] to archive 2024-11-27T11:36:27,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:36:27,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/08752b1a91054eb98e3897fe437f8dc0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/08752b1a91054eb98e3897fe437f8dc0 2024-11-27T11:36:27,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/b738e10da49b4e15bb1452e99727ecf0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/b738e10da49b4e15bb1452e99727ecf0 2024-11-27T11:36:27,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/3851716dc0ae47828b5d81d6419b4b7a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/3851716dc0ae47828b5d81d6419b4b7a 2024-11-27T11:36:27,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/60b967beec3049559485a47594339ee7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/60b967beec3049559485a47594339ee7 2024-11-27T11:36:27,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7e7fc3cc1afc412b8d98b6022daa01c4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7e7fc3cc1afc412b8d98b6022daa01c4 2024-11-27T11:36:27,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7267326b54fd4662b40f66751f42f0b2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/7267326b54fd4662b40f66751f42f0b2 2024-11-27T11:36:27,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/c32c36e4377046a281e3efbd08cccdd2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/c32c36e4377046a281e3efbd08cccdd2 2024-11-27T11:36:27,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d418e822a22c4e0190b19c2987901cfc to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d418e822a22c4e0190b19c2987901cfc 2024-11-27T11:36:27,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/2606bb3776d34f2aa610b843632fad5f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/2606bb3776d34f2aa610b843632fad5f 2024-11-27T11:36:27,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/9e7bcc46612842ecb6f31d0b2bee5bf4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/9e7bcc46612842ecb6f31d0b2bee5bf4 2024-11-27T11:36:27,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ee24fcc9a16f4097942957db48717dae to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ee24fcc9a16f4097942957db48717dae 2024-11-27T11:36:27,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/783352c93fb444bb99dea1e5bd71798d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/783352c93fb444bb99dea1e5bd71798d 2024-11-27T11:36:27,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/599d72ab808143b5bcbc9366219a1107 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/599d72ab808143b5bcbc9366219a1107 2024-11-27T11:36:27,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/5f0d8783c34c498ea784341ac25d262e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/5f0d8783c34c498ea784341ac25d262e 2024-11-27T11:36:27,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ddbae050f26d43ef990fad57685a1501 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/ddbae050f26d43ef990fad57685a1501 2024-11-27T11:36:27,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/770f76b252574a8c9e9025367820afb0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/770f76b252574a8c9e9025367820afb0 2024-11-27T11:36:27,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d086fa513ea0485a8d2e85e9a59183fc to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/d086fa513ea0485a8d2e85e9a59183fc 2024-11-27T11:36:27,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6d479dc920ae48b783443ccc7b525214 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6d479dc920ae48b783443ccc7b525214 2024-11-27T11:36:27,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6c753f0ae8f94535b50c7182e3201344 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6c753f0ae8f94535b50c7182e3201344 2024-11-27T11:36:27,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/87b1123acc1a4e34be3184961a81e99e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/87b1123acc1a4e34be3184961a81e99e 2024-11-27T11:36:27,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f67623e76f794ed0aa0681c46f8e3fef to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f67623e76f794ed0aa0681c46f8e3fef 2024-11-27T11:36:27,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/6b0a2a3caaf349e9bd2e458d7ea0f3f2 2024-11-27T11:36:27,092 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/recovered.edits/342.seqid, newMaxSeqId=342, maxSeqId=1 2024-11-27T11:36:27,093 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. 2024-11-27T11:36:27,093 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for 5e109c5702a251266cfb057894b39f84: Waiting for close lock at 1732707385155Running coprocessor pre-close hooks at 1732707385155Disabling compacts and flushes for region at 1732707385155Disabling writes for close at 1732707385779 (+624 ms)Obtaining lock to block concurrent updates at 1732707385779Preparing flush snapshotting stores in 5e109c5702a251266cfb057894b39f84 at 1732707385779Finished memstore snapshotting TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1732707385780 (+1 ms)Flushing stores of TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84. at 1732707385780Flushing 5e109c5702a251266cfb057894b39f84/A: creating writer at 1732707385780Flushing 5e109c5702a251266cfb057894b39f84/A: appending metadata at 1732707385782 (+2 ms)Flushing 5e109c5702a251266cfb057894b39f84/A: closing flushed file at 1732707385782Flushing 5e109c5702a251266cfb057894b39f84/B: creating writer at 1732707386189 (+407 ms)Flushing 5e109c5702a251266cfb057894b39f84/B: appending metadata at 1732707386192 (+3 ms)Flushing 5e109c5702a251266cfb057894b39f84/B: closing flushed file at 1732707386192Flushing 5e109c5702a251266cfb057894b39f84/C: creating writer at 1732707386599 (+407 ms)Flushing 5e109c5702a251266cfb057894b39f84/C: appending metadata at 1732707386601 (+2 ms)Flushing 5e109c5702a251266cfb057894b39f84/C: closing flushed file at 1732707386601Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@444afaa6: reopening flushed file at 1732707387009 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3111c020: reopening flushed file at 1732707387012 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a09db9c: reopening flushed file at 1732707387016 (+4 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 5e109c5702a251266cfb057894b39f84 in 1242ms, sequenceid=339, compaction requested=true at 1732707387021 (+5 ms)Writing region close event to WAL at 1732707387089 (+68 ms)Running coprocessor post-close hooks at 1732707387093 (+4 ms)Closed at 1732707387093 2024-11-27T11:36:27,095 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed 5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:27,096 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=5e109c5702a251266cfb057894b39f84, regionState=CLOSED 2024-11-27T11:36:27,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:36:27,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-27T11:36:27,099 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure 5e109c5702a251266cfb057894b39f84, server=5b617c7edc8e,35461,1732707299005 in 2.0960 sec 2024-11-27T11:36:27,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-27T11:36:27,101 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5e109c5702a251266cfb057894b39f84, UNASSIGN in 2.0990 sec 2024-11-27T11:36:27,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-27T11:36:27,102 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.1020 sec 2024-11-27T11:36:27,103 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707387103"}]},"ts":"1732707387103"} 2024-11-27T11:36:27,105 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T11:36:27,105 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T11:36:27,106 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.1110 sec 2024-11-27T11:36:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-27T11:36:27,153 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:27,154 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T11:36:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,155 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T11:36:27,156 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,159 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:27,161 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/recovered.edits] 2024-11-27T11:36:27,164 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/29e260c325104c3481fe312e55a705ae to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/29e260c325104c3481fe312e55a705ae 2024-11-27T11:36:27,165 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/58d7f5fb8e254a938deb86cd0e2033ec to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/58d7f5fb8e254a938deb86cd0e2033ec 2024-11-27T11:36:27,166 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d5140582b32d47a38af46021461a8ba2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/A/d5140582b32d47a38af46021461a8ba2 2024-11-27T11:36:27,168 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/45caed7b8d6649b08d2ccc87ee03af99 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/45caed7b8d6649b08d2ccc87ee03af99 2024-11-27T11:36:27,169 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/7fd82d70a4f240818207fb0f3c040cef to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/7fd82d70a4f240818207fb0f3c040cef 2024-11-27T11:36:27,171 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b57d01b4d8744a9e951b8f7c3bd981aa to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/B/b57d01b4d8744a9e951b8f7c3bd981aa 2024-11-27T11:36:27,173 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/563d6ac52e0f43648c526b9972b26dbf to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/563d6ac52e0f43648c526b9972b26dbf 2024-11-27T11:36:27,174 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/e1d85dfaf5d2488a92b2f5c31546c550 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/e1d85dfaf5d2488a92b2f5c31546c550 2024-11-27T11:36:27,175 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f071ded02b454c8fbd1669ab9012542e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/C/f071ded02b454c8fbd1669ab9012542e 2024-11-27T11:36:27,177 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/recovered.edits/342.seqid to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84/recovered.edits/342.seqid 2024-11-27T11:36:27,178 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/5e109c5702a251266cfb057894b39f84 2024-11-27T11:36:27,178 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T11:36:27,180 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,182 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T11:36:27,184 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T11:36:27,185 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,185 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T11:36:27,185 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732707387185"}]},"ts":"9223372036854775807"} 2024-11-27T11:36:27,187 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T11:36:27,187 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5e109c5702a251266cfb057894b39f84, NAME => 'TestAcidGuarantees,,1732707361701.5e109c5702a251266cfb057894b39f84.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T11:36:27,187 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T11:36:27,187 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732707387187"}]},"ts":"9223372036854775807"} 2024-11-27T11:36:27,189 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T11:36:27,189 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-11-27T11:36:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-27T11:36:27,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T11:36:27,263 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:27,272 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_612182728_22 at /127.0.0.1:49244 [Waiting for operation #295] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1287702838_22 at /127.0.0.1:59374 [Waiting for operation #164] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=304 (was 301) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6372 (was 6443) 2024-11-27T11:36:27,280 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=6371 2024-11-27T11:36:27,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:36:27,281 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:36:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:27,283 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T11:36:27,283 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:27,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-27T11:36:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-27T11:36:27,284 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T11:36:27,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742138_1314 (size=959) 2024-11-27T11:36:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-27T11:36:27,432 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T11:36:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-27T11:36:27,691 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:36:27,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742139_1315 (size=53) 2024-11-27T11:36:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-27T11:36:28,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:36:28,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f7a3b830a30a75170b3af25b3b7b615a, disabling compactions & flushes 2024-11-27T11:36:28,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. after waiting 0 ms 2024-11-27T11:36:28,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,096 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f7a3b830a30a75170b3af25b3b7b615a: Waiting for close lock at 1732707388096Disabling compacts and flushes for region at 1732707388096Disabling writes for close at 1732707388096Writing region close event to WAL at 1732707388096Closed at 1732707388096 2024-11-27T11:36:28,097 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T11:36:28,098 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732707388097"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732707388097"}]},"ts":"1732707388097"} 2024-11-27T11:36:28,099 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T11:36:28,100 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T11:36:28,100 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707388100"}]},"ts":"1732707388100"} 2024-11-27T11:36:28,101 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T11:36:28,101 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, ASSIGN}] 2024-11-27T11:36:28,102 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, ASSIGN 2024-11-27T11:36:28,102 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, ASSIGN; state=OFFLINE, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=false 2024-11-27T11:36:28,253 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:28,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, ASSIGN because future has completed 2024-11-27T11:36:28,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:36:28,409 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,409 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:36:28,409 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,409 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:36:28,409 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,409 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,411 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,412 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a3b830a30a75170b3af25b3b7b615a columnFamilyName A 2024-11-27T11:36:28,412 DEBUG [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:28,412 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(327): Store=f7a3b830a30a75170b3af25b3b7b615a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:28,412 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,413 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a3b830a30a75170b3af25b3b7b615a columnFamilyName B 2024-11-27T11:36:28,413 DEBUG [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:28,414 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(327): Store=f7a3b830a30a75170b3af25b3b7b615a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:28,414 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,415 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a3b830a30a75170b3af25b3b7b615a columnFamilyName C 2024-11-27T11:36:28,415 DEBUG [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:28,415 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(327): Store=f7a3b830a30a75170b3af25b3b7b615a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:28,415 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,416 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,416 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,417 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,417 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,417 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:36:28,418 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,420 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:36:28,420 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened f7a3b830a30a75170b3af25b3b7b615a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75477007, jitterRate=0.12469504773616791}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:36:28,420 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:28,421 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for f7a3b830a30a75170b3af25b3b7b615a: Running coprocessor pre-open hook at 1732707388409Writing region info on filesystem at 1732707388410 (+1 ms)Initializing all the Stores at 1732707388410Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707388410Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707388410Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707388410Cleaning up temporary data from old regions at 1732707388417 (+7 ms)Running coprocessor post-open hooks at 1732707388420 (+3 ms)Region opened successfully at 1732707388421 (+1 ms) 2024-11-27T11:36:28,421 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., pid=85, masterSystemTime=1732707388406 2024-11-27T11:36:28,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-27T11:36:28,423 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,423 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:28,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:28,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:36:28,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-27T11:36:28,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 in 170 msec 2024-11-27T11:36:28,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-27T11:36:28,428 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, ASSIGN in 325 msec 2024-11-27T11:36:28,428 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T11:36:28,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707388428"}]},"ts":"1732707388428"} 2024-11-27T11:36:28,429 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T11:36:28,430 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T11:36:28,431 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1490 sec 2024-11-27T11:36:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-27T11:36:29,433 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:29,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@791ba79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:29,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:29,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:29,435 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:29,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5cf73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:29,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:29,437 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:29,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6023397a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:29,437 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:29,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:29,439 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:29,440 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:36:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:36:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ac4b1b7 2024-11-27T11:36:29,440 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:36:29,441 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59226, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:36:29,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:36:29,442 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:36:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T11:36:29,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742140_1316 (size=995) 2024-11-27T11:36:29,853 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T11:36:29,853 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T11:36:29,855 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:36:29,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, REOPEN/MOVE}] 2024-11-27T11:36:29,858 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, REOPEN/MOVE 2024-11-27T11:36:29,858 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:29,860 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, REOPEN/MOVE because future has completed 2024-11-27T11:36:29,860 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:36:29,860 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:36:30,012 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,012 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:36:30,012 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing f7a3b830a30a75170b3af25b3b7b615a, disabling compactions & flushes 2024-11-27T11:36:30,012 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,012 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,012 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. after waiting 0 ms 2024-11-27T11:36:30,012 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,015 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T11:36:30,016 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,016 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for f7a3b830a30a75170b3af25b3b7b615a: Waiting for close lock at 1732707390012Running coprocessor pre-close hooks at 1732707390012Disabling compacts and flushes for region at 1732707390012Disabling writes for close at 1732707390012Writing region close event to WAL at 1732707390012Running coprocessor post-close hooks at 1732707390016 (+4 ms)Closed at 1732707390016 2024-11-27T11:36:30,016 WARN [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: f7a3b830a30a75170b3af25b3b7b615a to self. 2024-11-27T11:36:30,018 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,018 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=CLOSED 2024-11-27T11:36:30,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:36:30,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-27T11:36:30,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 in 160 msec 2024-11-27T11:36:30,022 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, REOPEN/MOVE; state=CLOSED, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=true 2024-11-27T11:36:30,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, REOPEN/MOVE because future has completed 2024-11-27T11:36:30,175 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:36:30,329 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,329 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:36:30,329 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,329 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:36:30,329 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,329 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,331 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,331 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a3b830a30a75170b3af25b3b7b615a columnFamilyName A 2024-11-27T11:36:30,332 DEBUG [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:30,332 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(327): Store=f7a3b830a30a75170b3af25b3b7b615a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:30,333 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,333 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a3b830a30a75170b3af25b3b7b615a columnFamilyName B 2024-11-27T11:36:30,333 DEBUG [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:30,334 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(327): Store=f7a3b830a30a75170b3af25b3b7b615a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:30,334 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,334 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f7a3b830a30a75170b3af25b3b7b615a columnFamilyName C 2024-11-27T11:36:30,334 DEBUG [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:30,335 INFO [StoreOpener-f7a3b830a30a75170b3af25b3b7b615a-1 {}] regionserver.HStore(327): Store=f7a3b830a30a75170b3af25b3b7b615a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:36:30,335 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,335 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,336 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,337 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,337 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,337 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:36:30,338 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,339 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened f7a3b830a30a75170b3af25b3b7b615a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66196293, jitterRate=-0.01359836757183075}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:36:30,339 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,339 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for f7a3b830a30a75170b3af25b3b7b615a: Running coprocessor pre-open hook at 1732707390330Writing region info on filesystem at 1732707390330Initializing all the Stores at 1732707390330Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707390330Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707390330Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707390330Cleaning up temporary data from old regions at 1732707390337 (+7 ms)Running coprocessor post-open hooks at 1732707390339 (+2 ms)Region opened successfully at 1732707390339 2024-11-27T11:36:30,340 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., pid=90, masterSystemTime=1732707390326 2024-11-27T11:36:30,342 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,342 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,342 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=OPEN, openSeqNum=5, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:36:30,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-27T11:36:30,346 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 in 170 msec 2024-11-27T11:36:30,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-27T11:36:30,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, REOPEN/MOVE in 489 msec 2024-11-27T11:36:30,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-27T11:36:30,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-27T11:36:30,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 907 msec 2024-11-27T11:36:30,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-27T11:36:30,352 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:30,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@152c3dc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,353 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47139b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,354 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,355 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c50c120, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ee8158b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,356 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7607bb8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,358 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ddc3dbb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@623abce5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,359 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40240625, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,360 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13ec49e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,361 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e247104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,362 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16ce28e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,363 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b5089f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,363 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12b0d422, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,364 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,364 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,364 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4da7dfd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fddb4cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,366 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615d5876, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,366 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,366 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,367 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77b8bd4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,368 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59330, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62117e55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,369 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b52a46, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,370 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e08d069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,371 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76fc89ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3214ae13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,372 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,372 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673a1449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,373 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,374 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a541b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,374 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c88af78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,374 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,374 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,375 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c6002d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,375 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,376 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59398, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,376 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a2f286b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,377 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a275aba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,377 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:36:30,377 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:36:30,378 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:36:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:36:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:36:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720ed243, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:36:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:36:30,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:30,379 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:36:30,379 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ada6d44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:36:30,380 DEBUG [Thread-1466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,380 DEBUG [Thread-1468 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,380 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,380 DEBUG [Thread-1455 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,380 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:30,381 DEBUG [Thread-1453 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,381 DEBUG [Thread-1449 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,381 DEBUG [Thread-1451 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,381 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,381 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,382 DEBUG [Thread-1457 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-27T11:36:30,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,382 DEBUG [Thread-1464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,383 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58010, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:36:30,384 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:36:30,384 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:30,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:30,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58050, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,384 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,385 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:36:30,385 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:30,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:30,386 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,394 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:36:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,405 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:30,405 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:36:30,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d878fd6702d640a494ea429e473ca2e6_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707390404/Put/seqid=0 2024-11-27T11:36:30,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742141_1317 (size=17034) 2024-11-27T11:36:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707450446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707450454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707450454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707450455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707450455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:30,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:30,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707450562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707450562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707450572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707450572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707450572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,581 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:30,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:30,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707450772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707450782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707450782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707450793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707450793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:30,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:30,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:30,842 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d878fd6702d640a494ea429e473ca2e6_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d878fd6702d640a494ea429e473ca2e6_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:30,843 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/717095c69b4546d88b2974ade5619897, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:30,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/717095c69b4546d88b2974ade5619897 is 175, key is test_row_0/A:col10/1732707390404/Put/seqid=0 2024-11-27T11:36:30,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742142_1318 (size=48139) 2024-11-27T11:36:30,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:30,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:30,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:31,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707451082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707451092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707451092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707451102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707451103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,110 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:31,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,252 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/717095c69b4546d88b2974ade5619897 2024-11-27T11:36:31,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/4ef43bde69294501b4f6492b6f86dcbf is 50, key is test_row_0/B:col10/1732707390404/Put/seqid=0 2024-11-27T11:36:31,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742143_1319 (size=12001) 2024-11-27T11:36:31,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/4ef43bde69294501b4f6492b6f86dcbf 2024-11-27T11:36:31,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:31,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/229ec81e50514bbe90014f22b1537da0 is 50, key is test_row_0/C:col10/1732707390404/Put/seqid=0 2024-11-27T11:36:31,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742144_1320 (size=12001) 2024-11-27T11:36:31,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:31,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,456 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:31,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,609 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:31,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707451602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707451602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707451602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707451613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,622 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707451622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:31,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:31,625 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:31,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/229ec81e50514bbe90014f22b1537da0 2024-11-27T11:36:31,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/717095c69b4546d88b2974ade5619897 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897 2024-11-27T11:36:31,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897, entries=250, sequenceid=16, filesize=47.0 K 2024-11-27T11:36:31,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/4ef43bde69294501b4f6492b6f86dcbf as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/4ef43bde69294501b4f6492b6f86dcbf 2024-11-27T11:36:31,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/4ef43bde69294501b4f6492b6f86dcbf, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T11:36:31,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/229ec81e50514bbe90014f22b1537da0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/229ec81e50514bbe90014f22b1537da0 2024-11-27T11:36:31,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/229ec81e50514bbe90014f22b1537da0, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T11:36:31,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f7a3b830a30a75170b3af25b3b7b615a in 1329ms, sequenceid=16, compaction requested=false 2024-11-27T11:36:31,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:31,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-27T11:36:31,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:31,761 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:36:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112701cca7eb86584f54aed9be10eb7d2edb_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707390452/Put/seqid=0 2024-11-27T11:36:31,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742145_1321 (size=12154) 2024-11-27T11:36:32,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,180 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112701cca7eb86584f54aed9be10eb7d2edb_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112701cca7eb86584f54aed9be10eb7d2edb_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:32,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/769a8f4c56d14263b6d76e77f0b9a9ec, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:32,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/769a8f4c56d14263b6d76e77f0b9a9ec is 175, key is test_row_0/A:col10/1732707390452/Put/seqid=0 2024-11-27T11:36:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742146_1322 (size=30955) 2024-11-27T11:36:32,187 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/769a8f4c56d14263b6d76e77f0b9a9ec 2024-11-27T11:36:32,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/649dfaab846e4ab88e958b9cc412d716 is 50, key is test_row_0/B:col10/1732707390452/Put/seqid=0 2024-11-27T11:36:32,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742147_1323 (size=12001) 2024-11-27T11:36:32,200 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/649dfaab846e4ab88e958b9cc412d716 2024-11-27T11:36:32,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/9e934b418b2d4d2f8a50240c317aae34 is 50, key is test_row_0/C:col10/1732707390452/Put/seqid=0 2024-11-27T11:36:32,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742148_1324 (size=12001) 2024-11-27T11:36:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:32,617 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/9e934b418b2d4d2f8a50240c317aae34 2024-11-27T11:36:32,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/769a8f4c56d14263b6d76e77f0b9a9ec as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec 2024-11-27T11:36:32,626 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:36:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:32,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:32,628 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T11:36:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/649dfaab846e4ab88e958b9cc412d716 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/649dfaab846e4ab88e958b9cc412d716 2024-11-27T11:36:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,635 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/649dfaab846e4ab88e958b9cc412d716, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:36:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/9e934b418b2d4d2f8a50240c317aae34 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/9e934b418b2d4d2f8a50240c317aae34 2024-11-27T11:36:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,640 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/9e934b418b2d4d2f8a50240c317aae34, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:36:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,641 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=20.13 KB/20610 for f7a3b830a30a75170b3af25b3b7b615a in 880ms, sequenceid=40, compaction requested=false 2024-11-27T11:36:32,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:32,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:32,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-27T11:36:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-27T11:36:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-27T11:36:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,645 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2580 sec 2024-11-27T11:36:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 2.2650 sec 2024-11-27T11:36:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:32,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:32,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:36:32,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127da5209052e4d4ba895ac8755318b7d2f_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:32,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742149_1325 (size=12154) 2024-11-27T11:36:32,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707452748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,750 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707452749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707452749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707452750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707452750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,761 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707452853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707452863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707452863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707452863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:32,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707452872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:32,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707453063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,071 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707453082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707453082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707453082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707453082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:33,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127da5209052e4d4ba895ac8755318b7d2f_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127da5209052e4d4ba895ac8755318b7d2f_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:33,119 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/c19c74f3022b487fac9b8b09b3b77039, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:33,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/c19c74f3022b487fac9b8b09b3b77039 is 175, key is test_row_0/A:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:33,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742150_1326 (size=30955) 2024-11-27T11:36:33,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707453383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707453392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707453392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707453392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,397 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707453402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,524 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/c19c74f3022b487fac9b8b09b3b77039 2024-11-27T11:36:33,531 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/a3cba248bf3b4f5a98e6cc3e3afe60bb is 50, key is test_row_0/B:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:33,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742151_1327 (size=12001) 2024-11-27T11:36:33,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707453903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707453903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707453903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707453903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,913 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,914 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707453922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:33,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:33,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/a3cba248bf3b4f5a98e6cc3e3afe60bb 2024-11-27T11:36:33,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/2e01c923c7504531b077a3f38aed493c is 50, key is test_row_0/C:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:33,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742152_1328 (size=12001) 2024-11-27T11:36:34,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/2e01c923c7504531b077a3f38aed493c 2024-11-27T11:36:34,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/c19c74f3022b487fac9b8b09b3b77039 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039 2024-11-27T11:36:34,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039, entries=150, sequenceid=52, filesize=30.2 K 2024-11-27T11:36:34,355 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/a3cba248bf3b4f5a98e6cc3e3afe60bb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/a3cba248bf3b4f5a98e6cc3e3afe60bb 2024-11-27T11:36:34,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/a3cba248bf3b4f5a98e6cc3e3afe60bb, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T11:36:34,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/2e01c923c7504531b077a3f38aed493c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/2e01c923c7504531b077a3f38aed493c 2024-11-27T11:36:34,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/2e01c923c7504531b077a3f38aed493c, entries=150, sequenceid=52, filesize=11.7 K 2024-11-27T11:36:34,363 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for f7a3b830a30a75170b3af25b3b7b615a in 1694ms, sequenceid=52, compaction requested=true 2024-11-27T11:36:34,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:34,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:34,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:34,363 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:34,363 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:34,364 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:34,364 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:34,364 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:34,364 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/A is initiating minor compaction (all files) 2024-11-27T11:36:34,364 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/B is initiating minor compaction (all files) 2024-11-27T11:36:34,365 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/B in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:34,365 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/A in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:34,365 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/4ef43bde69294501b4f6492b6f86dcbf, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/649dfaab846e4ab88e958b9cc412d716, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/a3cba248bf3b4f5a98e6cc3e3afe60bb] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.2 K 2024-11-27T11:36:34,365 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=107.5 K 2024-11-27T11:36:34,365 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:34,365 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039] 2024-11-27T11:36:34,365 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ef43bde69294501b4f6492b6f86dcbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732707390394 2024-11-27T11:36:34,365 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 717095c69b4546d88b2974ade5619897, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732707390394 2024-11-27T11:36:34,365 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 769a8f4c56d14263b6d76e77f0b9a9ec, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707390452 2024-11-27T11:36:34,365 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 649dfaab846e4ab88e958b9cc412d716, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707390452 2024-11-27T11:36:34,366 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19c74f3022b487fac9b8b09b3b77039, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707392666 2024-11-27T11:36:34,366 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a3cba248bf3b4f5a98e6cc3e3afe60bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707392666 2024-11-27T11:36:34,373 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:34,374 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/7173ec281ae243bcab897c38c9e6f62e is 50, key is test_row_0/B:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:34,377 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:34,382 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127c1bd4835245d496998dea0d551e03557_f7a3b830a30a75170b3af25b3b7b615a store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:34,385 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127c1bd4835245d496998dea0d551e03557_f7a3b830a30a75170b3af25b3b7b615a, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:34,385 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c1bd4835245d496998dea0d551e03557_f7a3b830a30a75170b3af25b3b7b615a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:34,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742153_1329 (size=12104) 2024-11-27T11:36:34,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742154_1330 (size=4469) 2024-11-27T11:36:34,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-27T11:36:34,552 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:34,553 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-27T11:36:34,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T11:36:34,554 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:34,555 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:34,555 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T11:36:34,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-27T11:36:34,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:34,711 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:36:34,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f3555c215fc34b7ead27a292ac022824_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707392746/Put/seqid=0 2024-11-27T11:36:34,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742155_1331 (size=12154) 2024-11-27T11:36:34,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:34,726 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127f3555c215fc34b7ead27a292ac022824_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3555c215fc34b7ead27a292ac022824_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:34,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/fca0072655db4c8c880afdde9231f127, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:34,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/fca0072655db4c8c880afdde9231f127 is 175, key is test_row_0/A:col10/1732707392746/Put/seqid=0 2024-11-27T11:36:34,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742156_1332 (size=30955) 2024-11-27T11:36:34,795 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/7173ec281ae243bcab897c38c9e6f62e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/7173ec281ae243bcab897c38c9e6f62e 2024-11-27T11:36:34,800 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#A#compaction#278 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:34,800 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/8048db9626f34c7182ed6cff21bd1b29 is 175, key is test_row_0/A:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:34,800 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/B of f7a3b830a30a75170b3af25b3b7b615a into 7173ec281ae243bcab897c38c9e6f62e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:34,801 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:34,801 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/B, priority=13, startTime=1732707394363; duration=0sec 2024-11-27T11:36:34,801 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:34,801 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:B 2024-11-27T11:36:34,801 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:34,802 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:34,802 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/C is initiating minor compaction (all files) 2024-11-27T11:36:34,802 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/C in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:34,802 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/229ec81e50514bbe90014f22b1537da0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/9e934b418b2d4d2f8a50240c317aae34, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/2e01c923c7504531b077a3f38aed493c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.2 K 2024-11-27T11:36:34,803 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 229ec81e50514bbe90014f22b1537da0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732707390394 2024-11-27T11:36:34,803 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e934b418b2d4d2f8a50240c317aae34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707390452 2024-11-27T11:36:34,804 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e01c923c7504531b077a3f38aed493c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707392666 2024-11-27T11:36:34,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742157_1333 (size=31058) 2024-11-27T11:36:34,812 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:34,812 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/379be4fae68b480bae6fecedfe8768af is 50, key is test_row_0/C:col10/1732707392667/Put/seqid=0 2024-11-27T11:36:34,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742158_1334 (size=12104) 2024-11-27T11:36:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T11:36:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:34,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707454940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:34,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707454941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:34,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707454941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,953 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:34,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707454943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707454943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:34,956 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:34,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707455053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707455053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707455063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707455063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,132 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/fca0072655db4c8c880afdde9231f127 2024-11-27T11:36:35,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/bf13dfc892314f7192e379481ed64461 is 50, key is test_row_0/B:col10/1732707392746/Put/seqid=0 2024-11-27T11:36:35,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742159_1335 (size=12001) 2024-11-27T11:36:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T11:36:35,209 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/8048db9626f34c7182ed6cff21bd1b29 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/8048db9626f34c7182ed6cff21bd1b29 2024-11-27T11:36:35,214 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/A of f7a3b830a30a75170b3af25b3b7b615a into 8048db9626f34c7182ed6cff21bd1b29(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:35,214 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:35,214 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/A, priority=13, startTime=1732707394363; duration=0sec 2024-11-27T11:36:35,214 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:35,214 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:A 2024-11-27T11:36:35,222 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/379be4fae68b480bae6fecedfe8768af as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/379be4fae68b480bae6fecedfe8768af 2024-11-27T11:36:35,226 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/C of f7a3b830a30a75170b3af25b3b7b615a into 379be4fae68b480bae6fecedfe8768af(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:35,226 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:35,226 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/C, priority=13, startTime=1732707394364; duration=0sec 2024-11-27T11:36:35,226 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:35,226 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:C 2024-11-27T11:36:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707455272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707455272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707455272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707455272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,543 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/bf13dfc892314f7192e379481ed64461 2024-11-27T11:36:35,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/191d053f49f74206bfb76f8062ad8b38 is 50, key is test_row_0/C:col10/1732707392746/Put/seqid=0 2024-11-27T11:36:35,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742160_1336 (size=12001) 2024-11-27T11:36:35,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707455583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707455592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707455592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707455592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:35,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T11:36:35,955 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/191d053f49f74206bfb76f8062ad8b38 2024-11-27T11:36:35,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/fca0072655db4c8c880afdde9231f127 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127 2024-11-27T11:36:35,963 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T11:36:35,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/bf13dfc892314f7192e379481ed64461 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/bf13dfc892314f7192e379481ed64461 2024-11-27T11:36:35,967 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/bf13dfc892314f7192e379481ed64461, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T11:36:35,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/191d053f49f74206bfb76f8062ad8b38 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/191d053f49f74206bfb76f8062ad8b38 2024-11-27T11:36:35,973 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/191d053f49f74206bfb76f8062ad8b38, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T11:36:35,973 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for f7a3b830a30a75170b3af25b3b7b615a in 1262ms, sequenceid=77, compaction requested=false 2024-11-27T11:36:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:35,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-27T11:36:35,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-27T11:36:35,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-27T11:36:35,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-27T11:36:35,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.4240 sec 2024-11-27T11:36:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:36,097 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:36,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:36:36,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279037a36b2dc24bed98c8b4bb80260509_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:36,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742161_1337 (size=14594) 2024-11-27T11:36:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707456153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,157 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707456154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707456156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707456157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707456262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,264 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707456272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707456273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707456273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,275 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707456472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707456482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707456483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707456483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,508 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:36,512 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279037a36b2dc24bed98c8b4bb80260509_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279037a36b2dc24bed98c8b4bb80260509_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:36,513 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/cf0ced93a1b8415fb656b6fbb312d8fc, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:36,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/cf0ced93a1b8415fb656b6fbb312d8fc is 175, key is test_row_0/A:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:36,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742162_1338 (size=39549) 2024-11-27T11:36:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-27T11:36:36,703 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:36,703 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-27T11:36:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:36,705 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:36,706 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:36,706 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:36,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707456792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707456792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,801 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707456793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:36,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707456803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:36,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:36,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:36,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:36,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:36,918 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/cf0ced93a1b8415fb656b6fbb312d8fc 2024-11-27T11:36:36,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/1bbadb2b53734a059cecdb8e4f43d9c8 is 50, key is test_row_0/B:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:36,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742163_1339 (size=12001) 2024-11-27T11:36:36,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:36,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707456962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:36,966 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:37,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:37,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:37,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:37,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:37,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707457312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707457312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:37,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707457312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:37,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/1bbadb2b53734a059cecdb8e4f43d9c8 2024-11-27T11:36:37,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:37,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707457322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:37,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:37,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/11505bb5ea2d41b19f45a60a6fe72450 is 50, key is test_row_0/C:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:37,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742164_1340 (size=12001) 2024-11-27T11:36:37,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:37,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:37,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/11505bb5ea2d41b19f45a60a6fe72450 2024-11-27T11:36:37,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/cf0ced93a1b8415fb656b6fbb312d8fc as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc 2024-11-27T11:36:37,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc, entries=200, sequenceid=92, filesize=38.6 K 2024-11-27T11:36:37,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/1bbadb2b53734a059cecdb8e4f43d9c8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/1bbadb2b53734a059cecdb8e4f43d9c8 2024-11-27T11:36:37,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/1bbadb2b53734a059cecdb8e4f43d9c8, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T11:36:37,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/11505bb5ea2d41b19f45a60a6fe72450 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/11505bb5ea2d41b19f45a60a6fe72450 2024-11-27T11:36:37,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/11505bb5ea2d41b19f45a60a6fe72450, entries=150, sequenceid=92, filesize=11.7 K 2024-11-27T11:36:37,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f7a3b830a30a75170b3af25b3b7b615a in 1662ms, sequenceid=92, compaction requested=true 2024-11-27T11:36:37,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:37,760 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:37,760 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:37,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:37,761 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:37,761 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:37,761 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/B is initiating minor compaction (all files) 2024-11-27T11:36:37,761 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/A is initiating minor compaction (all files) 2024-11-27T11:36:37,761 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/B in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,762 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/A in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,762 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/7173ec281ae243bcab897c38c9e6f62e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/bf13dfc892314f7192e379481ed64461, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/1bbadb2b53734a059cecdb8e4f43d9c8] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.3 K 2024-11-27T11:36:37,762 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/8048db9626f34c7182ed6cff21bd1b29, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=99.2 K 2024-11-27T11:36:37,762 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,762 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/8048db9626f34c7182ed6cff21bd1b29, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc] 2024-11-27T11:36:37,762 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7173ec281ae243bcab897c38c9e6f62e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707392666 2024-11-27T11:36:37,762 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8048db9626f34c7182ed6cff21bd1b29, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707392666 2024-11-27T11:36:37,762 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting bf13dfc892314f7192e379481ed64461, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707392746 2024-11-27T11:36:37,762 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fca0072655db4c8c880afdde9231f127, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707392746 2024-11-27T11:36:37,762 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bbadb2b53734a059cecdb8e4f43d9c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707394939 2024-11-27T11:36:37,763 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf0ced93a1b8415fb656b6fbb312d8fc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707394939 2024-11-27T11:36:37,773 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:37,774 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/6f474f5a02834c86b27d30b77385b582 is 50, key is test_row_0/B:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:37,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-27T11:36:37,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:37,777 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:36:37,784 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:37,787 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127ff02e4a8f52c4410bda63bca06f14db8_f7a3b830a30a75170b3af25b3b7b615a store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:37,789 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127ff02e4a8f52c4410bda63bca06f14db8_f7a3b830a30a75170b3af25b3b7b615a, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:37,790 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127ff02e4a8f52c4410bda63bca06f14db8_f7a3b830a30a75170b3af25b3b7b615a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:37,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742165_1341 (size=12207) 2024-11-27T11:36:37,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c3f706dd1eb743d38eddcf49bf3685ee_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707396154/Put/seqid=0 2024-11-27T11:36:37,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742166_1342 (size=4469) 2024-11-27T11:36:37,804 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#A#compaction#287 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:37,804 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/c6616ad325e94ec39ea7d7b96cd5add2 is 175, key is test_row_0/A:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742167_1343 (size=12154) 2024-11-27T11:36:37,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742168_1344 (size=31161) 2024-11-27T11:36:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:38,195 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/6f474f5a02834c86b27d30b77385b582 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/6f474f5a02834c86b27d30b77385b582 2024-11-27T11:36:38,200 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/B of f7a3b830a30a75170b3af25b3b7b615a into 6f474f5a02834c86b27d30b77385b582(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:38,200 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:38,200 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/B, priority=13, startTime=1732707397760; duration=0sec 2024-11-27T11:36:38,200 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:38,200 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:B 2024-11-27T11:36:38,200 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:38,201 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:38,201 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/C is initiating minor compaction (all files) 2024-11-27T11:36:38,201 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/C in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:38,201 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/379be4fae68b480bae6fecedfe8768af, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/191d053f49f74206bfb76f8062ad8b38, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/11505bb5ea2d41b19f45a60a6fe72450] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.3 K 2024-11-27T11:36:38,201 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 379be4fae68b480bae6fecedfe8768af, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732707392666 2024-11-27T11:36:38,202 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 191d053f49f74206bfb76f8062ad8b38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707392746 2024-11-27T11:36:38,202 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 11505bb5ea2d41b19f45a60a6fe72450, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707394939 2024-11-27T11:36:38,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:38,209 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#C#compaction#289 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:38,209 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/4bd7777d997f4f328b1f0deb453582ab is 50, key is test_row_0/C:col10/1732707394939/Put/seqid=0 2024-11-27T11:36:38,209 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127c3f706dd1eb743d38eddcf49bf3685ee_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c3f706dd1eb743d38eddcf49bf3685ee_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/0ac55b8a40f24699896563135631fa6a, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:38,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/0ac55b8a40f24699896563135631fa6a is 175, key is test_row_0/A:col10/1732707396154/Put/seqid=0 2024-11-27T11:36:38,217 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/c6616ad325e94ec39ea7d7b96cd5add2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c6616ad325e94ec39ea7d7b96cd5add2 2024-11-27T11:36:38,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742169_1345 (size=12207) 2024-11-27T11:36:38,222 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/A of f7a3b830a30a75170b3af25b3b7b615a into c6616ad325e94ec39ea7d7b96cd5add2(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:38,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:38,222 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/A, priority=13, startTime=1732707397759; duration=0sec 2024-11-27T11:36:38,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:38,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:A 2024-11-27T11:36:38,223 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/4bd7777d997f4f328b1f0deb453582ab as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/4bd7777d997f4f328b1f0deb453582ab 2024-11-27T11:36:38,228 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/C of f7a3b830a30a75170b3af25b3b7b615a into 4bd7777d997f4f328b1f0deb453582ab(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:38,228 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:38,228 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/C, priority=13, startTime=1732707397760; duration=0sec 2024-11-27T11:36:38,228 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:38,228 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:C 2024-11-27T11:36:38,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742170_1346 (size=30955) 2024-11-27T11:36:38,234 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/0ac55b8a40f24699896563135631fa6a 2024-11-27T11:36:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/449e2ce8dc4c475d8ab31aac5c74ca92 is 50, key is test_row_0/B:col10/1732707396154/Put/seqid=0 2024-11-27T11:36:38,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742171_1347 (size=12001) 2024-11-27T11:36:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:38,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:38,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707458342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707458343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,356 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707458347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707458349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707458462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707458462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707458472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,648 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/449e2ce8dc4c475d8ab31aac5c74ca92 2024-11-27T11:36:38,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/829a7b1a49a34a26962c90a2694db209 is 50, key is test_row_0/C:col10/1732707396154/Put/seqid=0 2024-11-27T11:36:38,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742172_1348 (size=12001) 2024-11-27T11:36:38,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707458672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707458672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707458683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707458992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707458992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:38,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707458992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:38,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,061 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/829a7b1a49a34a26962c90a2694db209 2024-11-27T11:36:39,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/0ac55b8a40f24699896563135631fa6a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a 2024-11-27T11:36:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a, entries=150, sequenceid=116, filesize=30.2 K 2024-11-27T11:36:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/449e2ce8dc4c475d8ab31aac5c74ca92 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/449e2ce8dc4c475d8ab31aac5c74ca92 2024-11-27T11:36:39,074 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/449e2ce8dc4c475d8ab31aac5c74ca92, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T11:36:39,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/829a7b1a49a34a26962c90a2694db209 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/829a7b1a49a34a26962c90a2694db209 2024-11-27T11:36:39,080 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/829a7b1a49a34a26962c90a2694db209, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T11:36:39,081 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f7a3b830a30a75170b3af25b3b7b615a in 1304ms, sequenceid=116, compaction requested=false 2024-11-27T11:36:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:39,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-27T11:36:39,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-27T11:36:39,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-27T11:36:39,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3770 sec 2024-11-27T11:36:39,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.3820 sec 2024-11-27T11:36:39,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:39,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:39,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:36:39,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127497b7d98114a499c8a2349fcdfb93f7e_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:39,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742173_1349 (size=12304) 2024-11-27T11:36:39,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707459557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707459559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707459560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707459662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,668 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707459673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707459673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707459872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707459882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707459883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:39,916 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:39,920 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127497b7d98114a499c8a2349fcdfb93f7e_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127497b7d98114a499c8a2349fcdfb93f7e_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:39,921 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/91ec5a76a23b48d68d1d74583c5f22e9, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:39,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/91ec5a76a23b48d68d1d74583c5f22e9 is 175, key is test_row_0/A:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:39,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742174_1350 (size=31105) 2024-11-27T11:36:40,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707460192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707460192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707460202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,211 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,326 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/91ec5a76a23b48d68d1d74583c5f22e9 2024-11-27T11:36:40,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/9c2dfc11c9fd434e9a33b2a8494d5b1b is 50, key is test_row_0/B:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:40,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742175_1351 (size=12151) 2024-11-27T11:36:40,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707460372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,382 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707460712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707460712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707460722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:40,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/9c2dfc11c9fd434e9a33b2a8494d5b1b 2024-11-27T11:36:40,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/023e0052fd144c34aeb354e1b2a37c6e is 50, key is test_row_0/C:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:40,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742176_1352 (size=12151) 2024-11-27T11:36:40,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-27T11:36:40,863 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:40,863 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:40,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-27T11:36:40,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T11:36:40,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:40,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:40,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:40,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T11:36:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:40,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707460983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:40,990 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:41,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T11:36:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:41,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:41,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:41,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/023e0052fd144c34aeb354e1b2a37c6e 2024-11-27T11:36:41,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/91ec5a76a23b48d68d1d74583c5f22e9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9 2024-11-27T11:36:41,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9, entries=150, sequenceid=132, filesize=30.4 K 2024-11-27T11:36:41,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/9c2dfc11c9fd434e9a33b2a8494d5b1b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c2dfc11c9fd434e9a33b2a8494d5b1b 2024-11-27T11:36:41,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c2dfc11c9fd434e9a33b2a8494d5b1b, entries=150, sequenceid=132, filesize=11.9 K 2024-11-27T11:36:41,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/023e0052fd144c34aeb354e1b2a37c6e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/023e0052fd144c34aeb354e1b2a37c6e 2024-11-27T11:36:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/023e0052fd144c34aeb354e1b2a37c6e, entries=150, sequenceid=132, filesize=11.9 K 2024-11-27T11:36:41,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7a3b830a30a75170b3af25b3b7b615a in 1662ms, sequenceid=132, compaction requested=true 2024-11-27T11:36:41,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:41,167 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:41,170 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:41,170 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/A is initiating minor compaction (all files) 2024-11-27T11:36:41,170 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/A in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,170 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c6616ad325e94ec39ea7d7b96cd5add2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=91.0 K 2024-11-27T11:36:41,170 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,170 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c6616ad325e94ec39ea7d7b96cd5add2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9] 2024-11-27T11:36:41,171 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c6616ad325e94ec39ea7d7b96cd5add2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707394939 2024-11-27T11:36:41,170 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:41,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:41,171 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ac55b8a40f24699896563135631fa6a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707396152 2024-11-27T11:36:41,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-27T11:36:41,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:41,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:41,172 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:41,172 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:41,172 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:36:41,172 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ec5a76a23b48d68d1d74583c5f22e9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732707398346 2024-11-27T11:36:41,175 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:41,175 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/B is initiating minor compaction (all files) 2024-11-27T11:36:41,175 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/B in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,175 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/6f474f5a02834c86b27d30b77385b582, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/449e2ce8dc4c475d8ab31aac5c74ca92, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c2dfc11c9fd434e9a33b2a8494d5b1b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.5 K 2024-11-27T11:36:41,176 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f474f5a02834c86b27d30b77385b582, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707394939 2024-11-27T11:36:41,177 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 449e2ce8dc4c475d8ab31aac5c74ca92, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707396152 2024-11-27T11:36:41,177 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c2dfc11c9fd434e9a33b2a8494d5b1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732707398346 2024-11-27T11:36:41,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T11:36:41,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d02955000f5548578e2596537fe619eb_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707399557/Put/seqid=0 2024-11-27T11:36:41,193 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:41,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742177_1353 (size=12304) 2024-11-27T11:36:41,202 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:41,203 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/559d42d579bd487eb2f6d5ac3463eef9 is 50, key is test_row_0/B:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:41,205 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411278d8b37096af24cffac4e49a27486389b_f7a3b830a30a75170b3af25b3b7b615a store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:41,206 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411278d8b37096af24cffac4e49a27486389b_f7a3b830a30a75170b3af25b3b7b615a, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:41,206 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278d8b37096af24cffac4e49a27486389b_f7a3b830a30a75170b3af25b3b7b615a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:41,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742178_1354 (size=12459) 2024-11-27T11:36:41,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742179_1355 (size=4469) 2024-11-27T11:36:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T11:36:41,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:41,602 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d02955000f5548578e2596537fe619eb_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d02955000f5548578e2596537fe619eb_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:41,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/d94f5387a5cf407abfa0be5d82de0489, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:41,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/d94f5387a5cf407abfa0be5d82de0489 is 175, key is test_row_0/A:col10/1732707399557/Put/seqid=0 2024-11-27T11:36:41,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742180_1356 (size=31105) 2024-11-27T11:36:41,620 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/559d42d579bd487eb2f6d5ac3463eef9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/559d42d579bd487eb2f6d5ac3463eef9 2024-11-27T11:36:41,621 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#A#compaction#296 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:41,622 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/f847b1f3c5d04c6ca9cd4f75e687eeac is 175, key is test_row_0/A:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:41,626 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/B of f7a3b830a30a75170b3af25b3b7b615a into 559d42d579bd487eb2f6d5ac3463eef9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:41,626 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:41,626 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/B, priority=13, startTime=1732707401171; duration=0sec 2024-11-27T11:36:41,626 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:41,626 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:B 2024-11-27T11:36:41,626 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:41,627 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:41,627 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/C is initiating minor compaction (all files) 2024-11-27T11:36:41,627 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/C in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:41,627 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/4bd7777d997f4f328b1f0deb453582ab, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/829a7b1a49a34a26962c90a2694db209, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/023e0052fd144c34aeb354e1b2a37c6e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.5 K 2024-11-27T11:36:41,628 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bd7777d997f4f328b1f0deb453582ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732707394939 2024-11-27T11:36:41,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742181_1357 (size=31413) 2024-11-27T11:36:41,629 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 829a7b1a49a34a26962c90a2694db209, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707396152 2024-11-27T11:36:41,629 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 023e0052fd144c34aeb354e1b2a37c6e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732707398346 2024-11-27T11:36:41,637 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#C#compaction#298 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:41,638 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/f7726ae56d8145fd93ff3dff5d3370d8 is 50, key is test_row_0/C:col10/1732707399504/Put/seqid=0 2024-11-27T11:36:41,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742182_1358 (size=12459) 2024-11-27T11:36:41,653 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/f7726ae56d8145fd93ff3dff5d3370d8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/f7726ae56d8145fd93ff3dff5d3370d8 2024-11-27T11:36:41,661 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/C of f7a3b830a30a75170b3af25b3b7b615a into f7726ae56d8145fd93ff3dff5d3370d8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:41,661 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:41,661 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/C, priority=13, startTime=1732707401171; duration=0sec 2024-11-27T11:36:41,661 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:41,662 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:C 2024-11-27T11:36:41,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:41,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:41,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707461766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:41,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707461766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:41,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:41,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707461767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707461873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:41,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707461873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,878 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:41,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707461882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:41,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T11:36:42,008 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/d94f5387a5cf407abfa0be5d82de0489 2024-11-27T11:36:42,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/0583d481a8ea4b0498db26afccdb5e71 is 50, key is test_row_0/B:col10/1732707399557/Put/seqid=0 2024-11-27T11:36:42,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742183_1359 (size=12151) 2024-11-27T11:36:42,033 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/f847b1f3c5d04c6ca9cd4f75e687eeac as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/f847b1f3c5d04c6ca9cd4f75e687eeac 2024-11-27T11:36:42,038 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/A of f7a3b830a30a75170b3af25b3b7b615a into f847b1f3c5d04c6ca9cd4f75e687eeac(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:42,038 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:42,038 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/A, priority=13, startTime=1732707401167; duration=0sec 2024-11-27T11:36:42,038 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:42,038 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:A 2024-11-27T11:36:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707462083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:42,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707462083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:42,091 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707462092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707462402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:42,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:42,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707462402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707462402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:42,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:42,421 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/0583d481a8ea4b0498db26afccdb5e71 2024-11-27T11:36:42,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/bb010795eecc473392213e48b38c31d7 is 50, key is test_row_0/C:col10/1732707399557/Put/seqid=0 2024-11-27T11:36:42,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742184_1360 (size=12151) 2024-11-27T11:36:42,838 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/bb010795eecc473392213e48b38c31d7 2024-11-27T11:36:42,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/d94f5387a5cf407abfa0be5d82de0489 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489 2024-11-27T11:36:42,847 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489, entries=150, sequenceid=155, filesize=30.4 K 2024-11-27T11:36:42,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/0583d481a8ea4b0498db26afccdb5e71 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0583d481a8ea4b0498db26afccdb5e71 2024-11-27T11:36:42,851 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0583d481a8ea4b0498db26afccdb5e71, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T11:36:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/bb010795eecc473392213e48b38c31d7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/bb010795eecc473392213e48b38c31d7 2024-11-27T11:36:42,855 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/bb010795eecc473392213e48b38c31d7, entries=150, sequenceid=155, filesize=11.9 K 2024-11-27T11:36:42,856 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f7a3b830a30a75170b3af25b3b7b615a in 1684ms, sequenceid=155, compaction requested=false 2024-11-27T11:36:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-27T11:36:42,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-27T11:36:42,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-27T11:36:42,859 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9910 sec 2024-11-27T11:36:42,861 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.9960 sec 2024-11-27T11:36:42,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:42,917 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:42,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-27T11:36:42,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278d5dad36f1d84fef984bdf24ff390d58_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:42,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742185_1361 (size=14794) 2024-11-27T11:36:42,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:42,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278d5dad36f1d84fef984bdf24ff390d58_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278d5dad36f1d84fef984bdf24ff390d58_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:42,933 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/e17ef9635f194aa4a51955433f79d753, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:42,934 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/e17ef9635f194aa4a51955433f79d753 is 175, key is test_row_0/A:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:42,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742186_1362 (size=39749) 2024-11-27T11:36:43,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-27T11:36:43,012 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:43,013 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-27T11:36:43,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:43,014 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:43,015 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:43,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707463010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707463011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707463016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:43,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707463132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707463132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707463132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:43,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:43,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:43,338 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/e17ef9635f194aa4a51955433f79d753 2024-11-27T11:36:43,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/e251167bf1de4200bb7dc37e38a5d815 is 50, key is test_row_0/B:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:43,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742187_1363 (size=12151) 2024-11-27T11:36:43,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707463342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,351 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707463352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707463353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,362 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:43,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707463652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707463672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707463672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:43,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:43,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/e251167bf1de4200bb7dc37e38a5d815 2024-11-27T11:36:43,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/d2cafb94d4c34fc9beabf5afa59b8c7c is 50, key is test_row_0/C:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:43,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742188_1364 (size=12151) 2024-11-27T11:36:43,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:43,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:43,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:44,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/d2cafb94d4c34fc9beabf5afa59b8c7c 2024-11-27T11:36:44,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/e17ef9635f194aa4a51955433f79d753 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753 2024-11-27T11:36:44,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753, entries=200, sequenceid=172, filesize=38.8 K 2024-11-27T11:36:44,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/e251167bf1de4200bb7dc37e38a5d815 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/e251167bf1de4200bb7dc37e38a5d815 2024-11-27T11:36:44,173 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/e251167bf1de4200bb7dc37e38a5d815, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T11:36:44,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/d2cafb94d4c34fc9beabf5afa59b8c7c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/d2cafb94d4c34fc9beabf5afa59b8c7c 2024-11-27T11:36:44,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707464162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/d2cafb94d4c34fc9beabf5afa59b8c7c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-27T11:36:44,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f7a3b830a30a75170b3af25b3b7b615a in 1261ms, sequenceid=172, compaction requested=true 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:44,178 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:44,178 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:44,178 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:44,179 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102267 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:44,179 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/A is initiating minor compaction (all files) 2024-11-27T11:36:44,179 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/A in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,179 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/f847b1f3c5d04c6ca9cd4f75e687eeac, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=99.9 K 2024-11-27T11:36:44,179 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,179 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/f847b1f3c5d04c6ca9cd4f75e687eeac, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753] 2024-11-27T11:36:44,180 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:44,180 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f847b1f3c5d04c6ca9cd4f75e687eeac, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732707398346 2024-11-27T11:36:44,180 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/B is initiating minor compaction (all files) 2024-11-27T11:36:44,180 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/B in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,180 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/559d42d579bd487eb2f6d5ac3463eef9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0583d481a8ea4b0498db26afccdb5e71, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/e251167bf1de4200bb7dc37e38a5d815] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.9 K 2024-11-27T11:36:44,180 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 559d42d579bd487eb2f6d5ac3463eef9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732707398346 2024-11-27T11:36:44,181 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d94f5387a5cf407abfa0be5d82de0489, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732707399546 2024-11-27T11:36:44,181 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0583d481a8ea4b0498db26afccdb5e71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732707399546 2024-11-27T11:36:44,181 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e17ef9635f194aa4a51955433f79d753, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732707401752 2024-11-27T11:36:44,181 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e251167bf1de4200bb7dc37e38a5d815, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732707401752 2024-11-27T11:36:44,190 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:44,191 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:44,191 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d276494e74b24f4596f3b6a91fe317f3_f7a3b830a30a75170b3af25b3b7b615a store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:44,192 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/0c8437c944054e30a1bdb154590b20c6 is 50, key is test_row_0/B:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:44,193 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d276494e74b24f4596f3b6a91fe317f3_f7a3b830a30a75170b3af25b3b7b615a, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:44,194 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d276494e74b24f4596f3b6a91fe317f3_f7a3b830a30a75170b3af25b3b7b615a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:44,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742189_1365 (size=12561) 2024-11-27T11:36:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:44,197 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:44,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:36:44,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742190_1366 (size=4469) 2024-11-27T11:36:44,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787fd0d7340754fa6a0a57c388b8a90a8_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707404196/Put/seqid=0 2024-11-27T11:36:44,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742191_1367 (size=14794) 2024-11-27T11:36:44,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707464233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707464234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,246 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707464343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707464352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58010 deadline: 1732707464402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,410 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:44,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:44,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707464563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707464563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,600 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#A#compaction#305 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:44,601 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/7aceab0599f7424eac609e7930ba5673 is 175, key is test_row_0/A:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:44,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/0c8437c944054e30a1bdb154590b20c6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0c8437c944054e30a1bdb154590b20c6 2024-11-27T11:36:44,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742192_1368 (size=31515) 2024-11-27T11:36:44,607 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:44,608 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/B of f7a3b830a30a75170b3af25b3b7b615a into 0c8437c944054e30a1bdb154590b20c6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:44,608 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:44,608 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/B, priority=13, startTime=1732707404178; duration=0sec 2024-11-27T11:36:44,608 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:44,608 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:B 2024-11-27T11:36:44,608 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:36:44,609 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:36:44,609 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/C is initiating minor compaction (all files) 2024-11-27T11:36:44,609 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/C in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,609 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/f7726ae56d8145fd93ff3dff5d3370d8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/bb010795eecc473392213e48b38c31d7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/d2cafb94d4c34fc9beabf5afa59b8c7c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=35.9 K 2024-11-27T11:36:44,610 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f7726ae56d8145fd93ff3dff5d3370d8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732707398346 2024-11-27T11:36:44,610 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting bb010795eecc473392213e48b38c31d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732707399546 2024-11-27T11:36:44,611 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d2cafb94d4c34fc9beabf5afa59b8c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732707401752 2024-11-27T11:36:44,611 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112787fd0d7340754fa6a0a57c388b8a90a8_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787fd0d7340754fa6a0a57c388b8a90a8_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:44,612 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/342c4fca117347a286d727aee1d89e80, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:44,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/342c4fca117347a286d727aee1d89e80 is 175, key is test_row_0/A:col10/1732707404196/Put/seqid=0 2024-11-27T11:36:44,618 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:44,619 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/b40f56ba9a89423685a49a44f65da9d2 is 50, key is test_row_0/C:col10/1732707401752/Put/seqid=0 2024-11-27T11:36:44,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742193_1369 (size=39749) 2024-11-27T11:36:44,622 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/342c4fca117347a286d727aee1d89e80 2024-11-27T11:36:44,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742194_1370 (size=12561) 2024-11-27T11:36:44,632 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/04e4e2c612e0419e9271c3cac1f62b25 is 50, key is test_row_0/B:col10/1732707404196/Put/seqid=0 2024-11-27T11:36:44,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742195_1371 (size=12151) 2024-11-27T11:36:44,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/04e4e2c612e0419e9271c3cac1f62b25 2024-11-27T11:36:44,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/72a8bbbffc004897b3488a0ca34799a9 is 50, key is test_row_0/C:col10/1732707404196/Put/seqid=0 2024-11-27T11:36:44,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742196_1372 (size=12151) 2024-11-27T11:36:44,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:44,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:44,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707464872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:44,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707464872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:44,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:45,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:45,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:45,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:36:45,011 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/7aceab0599f7424eac609e7930ba5673 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/7aceab0599f7424eac609e7930ba5673 2024-11-27T11:36:45,015 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/A of f7a3b830a30a75170b3af25b3b7b615a into 7aceab0599f7424eac609e7930ba5673(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:45,015 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:45,015 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/A, priority=13, startTime=1732707404178; duration=0sec 2024-11-27T11:36:45,015 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:45,015 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:A 2024-11-27T11:36:45,035 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/b40f56ba9a89423685a49a44f65da9d2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/b40f56ba9a89423685a49a44f65da9d2 2024-11-27T11:36:45,039 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/C of f7a3b830a30a75170b3af25b3b7b615a into b40f56ba9a89423685a49a44f65da9d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:45,039 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:45,040 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/C, priority=13, startTime=1732707404178; duration=0sec 2024-11-27T11:36:45,040 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:45,040 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:C 2024-11-27T11:36:45,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/72a8bbbffc004897b3488a0ca34799a9 2024-11-27T11:36:45,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/342c4fca117347a286d727aee1d89e80 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80 2024-11-27T11:36:45,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80, entries=200, sequenceid=195, filesize=38.8 K 2024-11-27T11:36:45,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/04e4e2c612e0419e9271c3cac1f62b25 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/04e4e2c612e0419e9271c3cac1f62b25 2024-11-27T11:36:45,060 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/04e4e2c612e0419e9271c3cac1f62b25, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T11:36:45,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/72a8bbbffc004897b3488a0ca34799a9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72a8bbbffc004897b3488a0ca34799a9 2024-11-27T11:36:45,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72a8bbbffc004897b3488a0ca34799a9, entries=150, sequenceid=195, filesize=11.9 K 2024-11-27T11:36:45,065 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f7a3b830a30a75170b3af25b3b7b615a in 867ms, sequenceid=195, compaction requested=false 2024-11-27T11:36:45,065 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:45,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-27T11:36:45,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:45,156 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:36:45,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278b61fa2f4db44cb896f9d3ab2e331cb8_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707404232/Put/seqid=0 2024-11-27T11:36:45,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742197_1373 (size=12304) 2024-11-27T11:36:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:45,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:45,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:45,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707465319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,327 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707465382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:45,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707465382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:45,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:45,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707465433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:45,575 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278b61fa2f4db44cb896f9d3ab2e331cb8_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278b61fa2f4db44cb896f9d3ab2e331cb8_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:45,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/71eab1645c044278a2e96309197c6e6c, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:45,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/71eab1645c044278a2e96309197c6e6c is 175, key is test_row_0/A:col10/1732707404232/Put/seqid=0 2024-11-27T11:36:45,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742198_1374 (size=31105) 2024-11-27T11:36:45,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707465642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:45,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707465953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:45,961 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:45,981 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/71eab1645c044278a2e96309197c6e6c 2024-11-27T11:36:45,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/25f94a71b5c948ea92580e811152ec30 is 50, key is test_row_0/B:col10/1732707404232/Put/seqid=0 2024-11-27T11:36:45,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742199_1375 (size=12151) 2024-11-27T11:36:46,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707466393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:46,396 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:46,397 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/25f94a71b5c948ea92580e811152ec30 2024-11-27T11:36:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:46,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/be702a7535c748d18874871b86f05d70 is 50, key is test_row_0/C:col10/1732707404232/Put/seqid=0 2024-11-27T11:36:46,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:46,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707466402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:46,409 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:46,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:46,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742200_1376 (size=12151) 2024-11-27T11:36:46,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:46,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707466473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:46,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:46,811 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/be702a7535c748d18874871b86f05d70 2024-11-27T11:36:46,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/71eab1645c044278a2e96309197c6e6c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c 2024-11-27T11:36:46,820 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c, entries=150, sequenceid=211, filesize=30.4 K 2024-11-27T11:36:46,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/25f94a71b5c948ea92580e811152ec30 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/25f94a71b5c948ea92580e811152ec30 2024-11-27T11:36:46,824 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/25f94a71b5c948ea92580e811152ec30, entries=150, sequenceid=211, filesize=11.9 K 2024-11-27T11:36:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/be702a7535c748d18874871b86f05d70 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/be702a7535c748d18874871b86f05d70 2024-11-27T11:36:46,830 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/be702a7535c748d18874871b86f05d70, entries=150, sequenceid=211, filesize=11.9 K 2024-11-27T11:36:46,831 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7a3b830a30a75170b3af25b3b7b615a in 1675ms, sequenceid=211, compaction requested=true 2024-11-27T11:36:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:46,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:46,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-27T11:36:46,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-27T11:36:46,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-27T11:36:46,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8180 sec 2024-11-27T11:36:46,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 3.8220 sec 2024-11-27T11:36:47,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-27T11:36:47,183 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:47,183 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:47,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-27T11:36:47,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T11:36:47,185 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:47,186 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:47,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:47,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T11:36:47,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-27T11:36:47,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:47,338 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:36:47,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271e8fe4a7099a49e9872e26073eb8611f_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707405255/Put/seqid=0 2024-11-27T11:36:47,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742201_1377 (size=12304) 2024-11-27T11:36:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:47,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:47,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T11:36:47,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:47,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707467558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:47,560 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:47,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:47,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707467662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:47,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:47,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:47,755 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411271e8fe4a7099a49e9872e26073eb8611f_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271e8fe4a7099a49e9872e26073eb8611f_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/4881c6b3c0674a18950a892fc0fe20cd, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:47,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/4881c6b3c0674a18950a892fc0fe20cd is 175, key is test_row_0/A:col10/1732707405255/Put/seqid=0 2024-11-27T11:36:47,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742202_1378 (size=31105) 2024-11-27T11:36:47,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T11:36:47,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:47,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707467882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:47,890 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:47,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:47,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:48,161 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/4881c6b3c0674a18950a892fc0fe20cd 2024-11-27T11:36:48,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/9c689e194bc74f0395b42a577cd50eb1 is 50, key is test_row_0/B:col10/1732707405255/Put/seqid=0 2024-11-27T11:36:48,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742203_1379 (size=12151) 2024-11-27T11:36:48,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:48,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707468202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:48,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T11:36:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58038 deadline: 1732707468422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:48,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:48,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58036 deadline: 1732707468422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:48,431 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,431 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:48,578 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/9c689e194bc74f0395b42a577cd50eb1 2024-11-27T11:36:48,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/81bcf8e4929c461ab3dd581a53cb7db8 is 50, key is test_row_0/C:col10/1732707405255/Put/seqid=0 2024-11-27T11:36:48,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742204_1380 (size=12151) 2024-11-27T11:36:48,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707468722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:48,728 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:48,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:48,990 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/81bcf8e4929c461ab3dd581a53cb7db8 2024-11-27T11:36:48,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/4881c6b3c0674a18950a892fc0fe20cd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd 2024-11-27T11:36:48,998 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd, entries=150, sequenceid=234, filesize=30.4 K 2024-11-27T11:36:48,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/9c689e194bc74f0395b42a577cd50eb1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c689e194bc74f0395b42a577cd50eb1 2024-11-27T11:36:49,002 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c689e194bc74f0395b42a577cd50eb1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T11:36:49,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/81bcf8e4929c461ab3dd581a53cb7db8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/81bcf8e4929c461ab3dd581a53cb7db8 2024-11-27T11:36:49,006 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/81bcf8e4929c461ab3dd581a53cb7db8, entries=150, sequenceid=234, filesize=11.9 K 2024-11-27T11:36:49,007 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f7a3b830a30a75170b3af25b3b7b615a in 1669ms, sequenceid=234, compaction requested=true 2024-11-27T11:36:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:49,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-27T11:36:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-27T11:36:49,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-27T11:36:49,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8220 sec 2024-11-27T11:36:49,012 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.8280 sec 2024-11-27T11:36:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-27T11:36:49,333 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:49,333 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:36:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-27T11:36:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T11:36:49,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:36:49,335 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:36:49,335 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:36:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T11:36:49,488 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-27T11:36:49,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:49,488 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:36:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127774af78a0950439aa8559623792d10c6_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707407533/Put/seqid=0 2024-11-27T11:36:49,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742205_1381 (size=12304) 2024-11-27T11:36:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T11:36:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:49,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. as already flushing 2024-11-27T11:36:49,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:49,903 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127774af78a0950439aa8559623792d10c6_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127774af78a0950439aa8559623792d10c6_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:49,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/39b4dd516fed40e69c58915c69171af6, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:49,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/39b4dd516fed40e69c58915c69171af6 is 175, key is test_row_0/A:col10/1732707407533/Put/seqid=0 2024-11-27T11:36:49,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742206_1382 (size=31105) 2024-11-27T11:36:49,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707469910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:49,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T11:36:50,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707470023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:50,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:50,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707470242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:50,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:50,309 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/39b4dd516fed40e69c58915c69171af6 2024-11-27T11:36:50,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/eba5b24035c2456aa1d50e3465636691 is 50, key is test_row_0/B:col10/1732707407533/Put/seqid=0 2024-11-27T11:36:50,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742207_1383 (size=12151) 2024-11-27T11:36:50,387 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-11-27T11:36:50,387 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:50,387 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,387 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,387 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:50,388 INFO [Thread-1468 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1468. 2024-11-27T11:36:50,388 DEBUG [Thread-1468 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:50,388 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,388 DEBUG [Thread-1468 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,388 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:50,391 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-11-27T11:36:50,391 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:50,391 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,391 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,391 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:50,397 INFO [Thread-1464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1464. 2024-11-27T11:36:50,397 DEBUG [Thread-1464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:50,397 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,397 DEBUG [Thread-1464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,397 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:50,399 INFO [Thread-1466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1466. 2024-11-27T11:36:50,399 DEBUG [Thread-1466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:50,399 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,399 DEBUG [Thread-1466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:50,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T11:36:50,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:50,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707470552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:50,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:50,720 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/eba5b24035c2456aa1d50e3465636691 2024-11-27T11:36:50,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/e12ca87b08d1447890ed428a9a1047d3 is 50, key is test_row_0/C:col10/1732707407533/Put/seqid=0 2024-11-27T11:36:50,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742208_1384 (size=12151) 2024-11-27T11:36:51,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58016 deadline: 1732707471053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:51,054 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_1' in f7a3b830a30a75170b3af25b3b7b615a of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18303 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:36:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:51,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:51,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:36:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58012 deadline: 1732707471062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:36:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:36:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:36:51,130 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/e12ca87b08d1447890ed428a9a1047d3 2024-11-27T11:36:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/39b4dd516fed40e69c58915c69171af6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6 2024-11-27T11:36:51,136 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6, entries=150, sequenceid=247, filesize=30.4 K 2024-11-27T11:36:51,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/eba5b24035c2456aa1d50e3465636691 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/eba5b24035c2456aa1d50e3465636691 2024-11-27T11:36:51,139 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/eba5b24035c2456aa1d50e3465636691, entries=150, sequenceid=247, filesize=11.9 K 2024-11-27T11:36:51,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/e12ca87b08d1447890ed428a9a1047d3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/e12ca87b08d1447890ed428a9a1047d3 2024-11-27T11:36:51,142 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/e12ca87b08d1447890ed428a9a1047d3, entries=150, sequenceid=247, filesize=11.9 K 2024-11-27T11:36:51,143 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f7a3b830a30a75170b3af25b3b7b615a in 1655ms, sequenceid=247, compaction requested=true 2024-11-27T11:36:51,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:51,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:51,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-27T11:36:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-27T11:36:51,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-27T11:36:51,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-27T11:36:51,147 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.8130 sec 2024-11-27T11:36:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-27T11:36:51,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:36:52,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:52,074 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:36:52,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:36:52,074 INFO [Thread-1455 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1455. 2024-11-27T11:36:52,074 DEBUG [Thread-1455 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:52,074 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:52,074 DEBUG [Thread-1455 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:52,074 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:52,079 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127596db033dd29490da78a36449ab2e332_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_0/A:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:52,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742209_1385 (size=12454) 2024-11-27T11:36:52,474 INFO [Thread-1449 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1449. 2024-11-27T11:36:52,474 DEBUG [Thread-1449 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:52,474 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:52,474 DEBUG [Thread-1449 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:52,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:52,483 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:36:52,484 INFO [Thread-1457 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1457. 2024-11-27T11:36:52,484 DEBUG [Thread-1457 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:52,484 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:52,484 DEBUG [Thread-1457 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:52,484 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:52,486 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127596db033dd29490da78a36449ab2e332_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127596db033dd29490da78a36449ab2e332_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:36:52,487 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/ee3bbf8be0e643289a6585ae243e35fe, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:52,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/ee3bbf8be0e643289a6585ae243e35fe is 175, key is test_row_0/A:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742210_1386 (size=31255) 2024-11-27T11:36:52,891 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/ee3bbf8be0e643289a6585ae243e35fe 2024-11-27T11:36:52,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/92bc5905a5f242388328a4d2160a43e5 is 50, key is test_row_0/B:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:52,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742211_1387 (size=12301) 2024-11-27T11:36:53,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/92bc5905a5f242388328a4d2160a43e5 2024-11-27T11:36:53,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/3f6a41ab0ad64a15aa7621196fed9369 is 50, key is test_row_0/C:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:53,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742212_1388 (size=12301) 2024-11-27T11:36:53,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/3f6a41ab0ad64a15aa7621196fed9369 2024-11-27T11:36:53,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/ee3bbf8be0e643289a6585ae243e35fe as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe 2024-11-27T11:36:53,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe, entries=150, sequenceid=271, filesize=30.5 K 2024-11-27T11:36:53,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/92bc5905a5f242388328a4d2160a43e5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/92bc5905a5f242388328a4d2160a43e5 2024-11-27T11:36:53,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/92bc5905a5f242388328a4d2160a43e5, entries=150, sequenceid=271, filesize=12.0 K 2024-11-27T11:36:53,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/3f6a41ab0ad64a15aa7621196fed9369 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/3f6a41ab0ad64a15aa7621196fed9369 2024-11-27T11:36:53,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/3f6a41ab0ad64a15aa7621196fed9369, entries=150, sequenceid=271, filesize=12.0 K 2024-11-27T11:36:53,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=13.42 KB/13740 for f7a3b830a30a75170b3af25b3b7b615a in 1651ms, sequenceid=271, compaction requested=true 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f7a3b830a30a75170b3af25b3b7b615a:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:36:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:53,725 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T11:36:53,725 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T11:36:53,726 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195834 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T11:36:53,726 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73466 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T11:36:53,726 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/B is initiating minor compaction (all files) 2024-11-27T11:36:53,726 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/A is initiating minor compaction (all files) 2024-11-27T11:36:53,726 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/B in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:53,726 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/A in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:53,726 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0c8437c944054e30a1bdb154590b20c6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/04e4e2c612e0419e9271c3cac1f62b25, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/25f94a71b5c948ea92580e811152ec30, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c689e194bc74f0395b42a577cd50eb1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/eba5b24035c2456aa1d50e3465636691, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/92bc5905a5f242388328a4d2160a43e5] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=71.7 K 2024-11-27T11:36:53,726 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/7aceab0599f7424eac609e7930ba5673, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=191.2 K 2024-11-27T11:36:53,726 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:53,726 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/7aceab0599f7424eac609e7930ba5673, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe] 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c8437c944054e30a1bdb154590b20c6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732707401752 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7aceab0599f7424eac609e7930ba5673, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732707401752 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e4e2c612e0419e9271c3cac1f62b25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732707403009 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 342c4fca117347a286d727aee1d89e80, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732707403009 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 25f94a71b5c948ea92580e811152ec30, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732707404208 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71eab1645c044278a2e96309197c6e6c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732707404208 2024-11-27T11:36:53,727 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c689e194bc74f0395b42a577cd50eb1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732707405255 2024-11-27T11:36:53,728 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4881c6b3c0674a18950a892fc0fe20cd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732707405255 2024-11-27T11:36:53,728 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting eba5b24035c2456aa1d50e3465636691, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732707407533 2024-11-27T11:36:53,728 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39b4dd516fed40e69c58915c69171af6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732707407533 2024-11-27T11:36:53,728 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bc5905a5f242388328a4d2160a43e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732707409871 2024-11-27T11:36:53,728 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee3bbf8be0e643289a6585ae243e35fe, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732707409871 2024-11-27T11:36:53,736 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:53,737 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112768af458082804230b20bed2958cbab33_f7a3b830a30a75170b3af25b3b7b615a store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:53,738 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:53,738 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/454398b3b9464bc6b999aa4c21e171c1 is 50, key is test_row_0/B:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:53,740 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112768af458082804230b20bed2958cbab33_f7a3b830a30a75170b3af25b3b7b615a, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:53,741 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112768af458082804230b20bed2958cbab33_f7a3b830a30a75170b3af25b3b7b615a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:36:53,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742213_1389 (size=12915) 2024-11-27T11:36:53,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742214_1390 (size=4469) 2024-11-27T11:36:54,146 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/454398b3b9464bc6b999aa4c21e171c1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/454398b3b9464bc6b999aa4c21e171c1 2024-11-27T11:36:54,146 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#A#compaction#323 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:54,147 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/bd9fc2e501fc493d9ee216b369cef561 is 175, key is test_row_0/A:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:54,150 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/B of f7a3b830a30a75170b3af25b3b7b615a into 454398b3b9464bc6b999aa4c21e171c1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:54,150 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:54,150 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/B, priority=10, startTime=1732707413725; duration=0sec 2024-11-27T11:36:54,150 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:36:54,150 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:B 2024-11-27T11:36:54,150 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T11:36:54,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742215_1391 (size=31869) 2024-11-27T11:36:54,151 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73466 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T11:36:54,151 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): f7a3b830a30a75170b3af25b3b7b615a/C is initiating minor compaction (all files) 2024-11-27T11:36:54,151 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f7a3b830a30a75170b3af25b3b7b615a/C in TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:36:54,151 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/b40f56ba9a89423685a49a44f65da9d2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72a8bbbffc004897b3488a0ca34799a9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/be702a7535c748d18874871b86f05d70, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/81bcf8e4929c461ab3dd581a53cb7db8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/e12ca87b08d1447890ed428a9a1047d3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/3f6a41ab0ad64a15aa7621196fed9369] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp, totalSize=71.7 K 2024-11-27T11:36:54,151 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b40f56ba9a89423685a49a44f65da9d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732707401752 2024-11-27T11:36:54,152 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a8bbbffc004897b3488a0ca34799a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732707403009 2024-11-27T11:36:54,152 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting be702a7535c748d18874871b86f05d70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732707404208 2024-11-27T11:36:54,152 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 81bcf8e4929c461ab3dd581a53cb7db8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732707405255 2024-11-27T11:36:54,152 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e12ca87b08d1447890ed428a9a1047d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732707407533 2024-11-27T11:36:54,153 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6a41ab0ad64a15aa7621196fed9369, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732707409871 2024-11-27T11:36:54,161 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f7a3b830a30a75170b3af25b3b7b615a#C#compaction#324 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:36:54,161 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/5242756e9c274bfc8f2d05ef902f64c2 is 50, key is test_row_0/C:col10/1732707409908/Put/seqid=0 2024-11-27T11:36:54,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742216_1392 (size=12915) 2024-11-27T11:36:54,514 INFO [Thread-1453 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1453. 2024-11-27T11:36:54,514 DEBUG [Thread-1453 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:36:54,514 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:54,514 DEBUG [Thread-1453 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:36:54,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:36:54,555 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/bd9fc2e501fc493d9ee216b369cef561 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/bd9fc2e501fc493d9ee216b369cef561 2024-11-27T11:36:54,558 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/A of f7a3b830a30a75170b3af25b3b7b615a into bd9fc2e501fc493d9ee216b369cef561(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:54,559 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:54,559 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/A, priority=10, startTime=1732707413725; duration=0sec 2024-11-27T11:36:54,559 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:54,559 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:A 2024-11-27T11:36:54,567 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/5242756e9c274bfc8f2d05ef902f64c2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/5242756e9c274bfc8f2d05ef902f64c2 2024-11-27T11:36:54,570 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in f7a3b830a30a75170b3af25b3b7b615a/C of f7a3b830a30a75170b3af25b3b7b615a into 5242756e9c274bfc8f2d05ef902f64c2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:36:54,570 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f7a3b830a30a75170b3af25b3b7b615a: 2024-11-27T11:36:54,570 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., storeName=f7a3b830a30a75170b3af25b3b7b615a/C, priority=10, startTime=1732707413725; duration=0sec 2024-11-27T11:36:54,571 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:36:54,571 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f7a3b830a30a75170b3af25b3b7b615a:C 2024-11-27T11:36:57,432 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T11:37:01,124 INFO [Thread-1451 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1451. 2024-11-27T11:37:01,124 DEBUG [Thread-1451 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:01,124 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:01,124 DEBUG [Thread-1451 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:01,124 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1219 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3646 rows 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1220 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3656 rows 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1228 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3679 rows 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3676 rows 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1223 2024-11-27T11:37:01,124 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3658 rows 2024-11-27T11:37:01,124 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:37:01,124 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:37:01,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:01,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:01,125 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:01,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T11:37:01,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:01,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T11:37:01,128 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707421127"}]},"ts":"1732707421127"} 2024-11-27T11:37:01,129 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T11:37:01,129 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T11:37:01,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:37:01,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, UNASSIGN}] 2024-11-27T11:37:01,131 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, UNASSIGN 2024-11-27T11:37:01,132 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:01,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, UNASSIGN because future has completed 2024-11-27T11:37:01,133 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:37:01,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:37:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T11:37:01,285 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(122): Close f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:01,285 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:37:01,285 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1722): Closing f7a3b830a30a75170b3af25b3b7b615a, disabling compactions & flushes 2024-11-27T11:37:01,285 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:37:01,285 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:37:01,285 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. after waiting 0 ms 2024-11-27T11:37:01,285 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:37:01,285 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(2902): Flushing f7a3b830a30a75170b3af25b3b7b615a 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T11:37:01,290 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eaa066a2a0fd427c8f7014daf170c6ec_f7a3b830a30a75170b3af25b3b7b615a is 50, key is test_row_1/A:col10/1732707421123/Put/seqid=0 2024-11-27T11:37:01,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742217_1393 (size=9914) 2024-11-27T11:37:01,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T11:37:01,694 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:01,697 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eaa066a2a0fd427c8f7014daf170c6ec_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eaa066a2a0fd427c8f7014daf170c6ec_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:01,698 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/3103f742edf446a380768898cccc0d76, store: [table=TestAcidGuarantees family=A region=f7a3b830a30a75170b3af25b3b7b615a] 2024-11-27T11:37:01,698 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/3103f742edf446a380768898cccc0d76 is 175, key is test_row_1/A:col10/1732707421123/Put/seqid=0 2024-11-27T11:37:01,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742218_1394 (size=22561) 2024-11-27T11:37:01,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T11:37:01,817 WARN [regionserver/5b617c7edc8e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-27T11:37:01,817 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/8048db9626f34c7182ed6cff21bd1b29, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c6616ad325e94ec39ea7d7b96cd5add2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/f847b1f3c5d04c6ca9cd4f75e687eeac, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/7aceab0599f7424eac609e7930ba5673, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe] to archive 2024-11-27T11:37:01,818 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:37:01,820 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/717095c69b4546d88b2974ade5619897 2024-11-27T11:37:01,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/769a8f4c56d14263b6d76e77f0b9a9ec 2024-11-27T11:37:01,821 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/8048db9626f34c7182ed6cff21bd1b29 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/8048db9626f34c7182ed6cff21bd1b29 2024-11-27T11:37:01,822 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c19c74f3022b487fac9b8b09b3b77039 2024-11-27T11:37:01,823 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/fca0072655db4c8c880afdde9231f127 2024-11-27T11:37:01,824 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/cf0ced93a1b8415fb656b6fbb312d8fc 2024-11-27T11:37:01,825 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c6616ad325e94ec39ea7d7b96cd5add2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/c6616ad325e94ec39ea7d7b96cd5add2 2024-11-27T11:37:01,826 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/0ac55b8a40f24699896563135631fa6a 2024-11-27T11:37:01,826 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/f847b1f3c5d04c6ca9cd4f75e687eeac to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/f847b1f3c5d04c6ca9cd4f75e687eeac 2024-11-27T11:37:01,827 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/91ec5a76a23b48d68d1d74583c5f22e9 2024-11-27T11:37:01,828 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/d94f5387a5cf407abfa0be5d82de0489 2024-11-27T11:37:01,829 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/e17ef9635f194aa4a51955433f79d753 2024-11-27T11:37:01,830 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/7aceab0599f7424eac609e7930ba5673 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/7aceab0599f7424eac609e7930ba5673 2024-11-27T11:37:01,831 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/342c4fca117347a286d727aee1d89e80 2024-11-27T11:37:01,832 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/71eab1645c044278a2e96309197c6e6c 2024-11-27T11:37:01,832 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/4881c6b3c0674a18950a892fc0fe20cd 2024-11-27T11:37:01,833 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/39b4dd516fed40e69c58915c69171af6 2024-11-27T11:37:01,834 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/ee3bbf8be0e643289a6585ae243e35fe 2024-11-27T11:37:01,836 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/4ef43bde69294501b4f6492b6f86dcbf, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/649dfaab846e4ab88e958b9cc412d716, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/7173ec281ae243bcab897c38c9e6f62e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/a3cba248bf3b4f5a98e6cc3e3afe60bb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/bf13dfc892314f7192e379481ed64461, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/6f474f5a02834c86b27d30b77385b582, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/1bbadb2b53734a059cecdb8e4f43d9c8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/449e2ce8dc4c475d8ab31aac5c74ca92, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/559d42d579bd487eb2f6d5ac3463eef9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c2dfc11c9fd434e9a33b2a8494d5b1b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0583d481a8ea4b0498db26afccdb5e71, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0c8437c944054e30a1bdb154590b20c6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/e251167bf1de4200bb7dc37e38a5d815, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/04e4e2c612e0419e9271c3cac1f62b25, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/25f94a71b5c948ea92580e811152ec30, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c689e194bc74f0395b42a577cd50eb1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/eba5b24035c2456aa1d50e3465636691, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/92bc5905a5f242388328a4d2160a43e5] to archive 2024-11-27T11:37:01,837 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:37:01,838 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/4ef43bde69294501b4f6492b6f86dcbf to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/4ef43bde69294501b4f6492b6f86dcbf 2024-11-27T11:37:01,839 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/649dfaab846e4ab88e958b9cc412d716 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/649dfaab846e4ab88e958b9cc412d716 2024-11-27T11:37:01,840 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/7173ec281ae243bcab897c38c9e6f62e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/7173ec281ae243bcab897c38c9e6f62e 2024-11-27T11:37:01,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/a3cba248bf3b4f5a98e6cc3e3afe60bb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/a3cba248bf3b4f5a98e6cc3e3afe60bb 2024-11-27T11:37:01,841 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/bf13dfc892314f7192e379481ed64461 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/bf13dfc892314f7192e379481ed64461 2024-11-27T11:37:01,842 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/6f474f5a02834c86b27d30b77385b582 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/6f474f5a02834c86b27d30b77385b582 2024-11-27T11:37:01,843 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/1bbadb2b53734a059cecdb8e4f43d9c8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/1bbadb2b53734a059cecdb8e4f43d9c8 2024-11-27T11:37:01,844 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/449e2ce8dc4c475d8ab31aac5c74ca92 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/449e2ce8dc4c475d8ab31aac5c74ca92 2024-11-27T11:37:01,844 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/559d42d579bd487eb2f6d5ac3463eef9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/559d42d579bd487eb2f6d5ac3463eef9 2024-11-27T11:37:01,845 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c2dfc11c9fd434e9a33b2a8494d5b1b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c2dfc11c9fd434e9a33b2a8494d5b1b 2024-11-27T11:37:01,846 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0583d481a8ea4b0498db26afccdb5e71 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0583d481a8ea4b0498db26afccdb5e71 2024-11-27T11:37:01,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0c8437c944054e30a1bdb154590b20c6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/0c8437c944054e30a1bdb154590b20c6 2024-11-27T11:37:01,847 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/e251167bf1de4200bb7dc37e38a5d815 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/e251167bf1de4200bb7dc37e38a5d815 2024-11-27T11:37:01,848 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/04e4e2c612e0419e9271c3cac1f62b25 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/04e4e2c612e0419e9271c3cac1f62b25 2024-11-27T11:37:01,849 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/25f94a71b5c948ea92580e811152ec30 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/25f94a71b5c948ea92580e811152ec30 2024-11-27T11:37:01,850 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c689e194bc74f0395b42a577cd50eb1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/9c689e194bc74f0395b42a577cd50eb1 2024-11-27T11:37:01,851 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/eba5b24035c2456aa1d50e3465636691 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/eba5b24035c2456aa1d50e3465636691 2024-11-27T11:37:01,851 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/92bc5905a5f242388328a4d2160a43e5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/92bc5905a5f242388328a4d2160a43e5 2024-11-27T11:37:01,854 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/229ec81e50514bbe90014f22b1537da0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/9e934b418b2d4d2f8a50240c317aae34, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/379be4fae68b480bae6fecedfe8768af, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/2e01c923c7504531b077a3f38aed493c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/191d053f49f74206bfb76f8062ad8b38, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/4bd7777d997f4f328b1f0deb453582ab, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/11505bb5ea2d41b19f45a60a6fe72450, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/829a7b1a49a34a26962c90a2694db209, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/f7726ae56d8145fd93ff3dff5d3370d8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/023e0052fd144c34aeb354e1b2a37c6e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/bb010795eecc473392213e48b38c31d7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/b40f56ba9a89423685a49a44f65da9d2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/d2cafb94d4c34fc9beabf5afa59b8c7c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72a8bbbffc004897b3488a0ca34799a9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/be702a7535c748d18874871b86f05d70, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/81bcf8e4929c461ab3dd581a53cb7db8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/e12ca87b08d1447890ed428a9a1047d3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/3f6a41ab0ad64a15aa7621196fed9369] to archive 2024-11-27T11:37:01,854 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:37:01,855 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/229ec81e50514bbe90014f22b1537da0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/229ec81e50514bbe90014f22b1537da0 2024-11-27T11:37:01,856 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/9e934b418b2d4d2f8a50240c317aae34 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/9e934b418b2d4d2f8a50240c317aae34 2024-11-27T11:37:01,857 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/379be4fae68b480bae6fecedfe8768af to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/379be4fae68b480bae6fecedfe8768af 2024-11-27T11:37:01,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/2e01c923c7504531b077a3f38aed493c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/2e01c923c7504531b077a3f38aed493c 2024-11-27T11:37:01,858 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/191d053f49f74206bfb76f8062ad8b38 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/191d053f49f74206bfb76f8062ad8b38 2024-11-27T11:37:01,859 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/4bd7777d997f4f328b1f0deb453582ab to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/4bd7777d997f4f328b1f0deb453582ab 2024-11-27T11:37:01,860 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/11505bb5ea2d41b19f45a60a6fe72450 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/11505bb5ea2d41b19f45a60a6fe72450 2024-11-27T11:37:01,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/829a7b1a49a34a26962c90a2694db209 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/829a7b1a49a34a26962c90a2694db209 2024-11-27T11:37:01,861 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/f7726ae56d8145fd93ff3dff5d3370d8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/f7726ae56d8145fd93ff3dff5d3370d8 2024-11-27T11:37:01,862 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/023e0052fd144c34aeb354e1b2a37c6e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/023e0052fd144c34aeb354e1b2a37c6e 2024-11-27T11:37:01,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/bb010795eecc473392213e48b38c31d7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/bb010795eecc473392213e48b38c31d7 2024-11-27T11:37:01,863 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/b40f56ba9a89423685a49a44f65da9d2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/b40f56ba9a89423685a49a44f65da9d2 2024-11-27T11:37:01,864 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/d2cafb94d4c34fc9beabf5afa59b8c7c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/d2cafb94d4c34fc9beabf5afa59b8c7c 2024-11-27T11:37:01,865 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72a8bbbffc004897b3488a0ca34799a9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72a8bbbffc004897b3488a0ca34799a9 2024-11-27T11:37:01,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/be702a7535c748d18874871b86f05d70 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/be702a7535c748d18874871b86f05d70 2024-11-27T11:37:01,866 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/81bcf8e4929c461ab3dd581a53cb7db8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/81bcf8e4929c461ab3dd581a53cb7db8 2024-11-27T11:37:01,867 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/e12ca87b08d1447890ed428a9a1047d3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/e12ca87b08d1447890ed428a9a1047d3 2024-11-27T11:37:01,868 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/5b617c7edc8e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/3f6a41ab0ad64a15aa7621196fed9369 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/3f6a41ab0ad64a15aa7621196fed9369 2024-11-27T11:37:02,102 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=281, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/3103f742edf446a380768898cccc0d76 2024-11-27T11:37:02,108 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/57da9a309b7b403a822fcfa79a9877af is 50, key is test_row_1/B:col10/1732707421123/Put/seqid=0 2024-11-27T11:37:02,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742219_1395 (size=9857) 2024-11-27T11:37:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T11:37:02,511 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/57da9a309b7b403a822fcfa79a9877af 2024-11-27T11:37:02,517 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/72217aae2cf24ef1aad0687af23da144 is 50, key is test_row_1/C:col10/1732707421123/Put/seqid=0 2024-11-27T11:37:02,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742220_1396 (size=9857) 2024-11-27T11:37:02,921 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/72217aae2cf24ef1aad0687af23da144 2024-11-27T11:37:02,924 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/A/3103f742edf446a380768898cccc0d76 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/3103f742edf446a380768898cccc0d76 2024-11-27T11:37:02,927 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/3103f742edf446a380768898cccc0d76, entries=100, sequenceid=281, filesize=22.0 K 2024-11-27T11:37:02,928 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/B/57da9a309b7b403a822fcfa79a9877af as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/57da9a309b7b403a822fcfa79a9877af 2024-11-27T11:37:02,931 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/57da9a309b7b403a822fcfa79a9877af, entries=100, sequenceid=281, filesize=9.6 K 2024-11-27T11:37:02,931 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/.tmp/C/72217aae2cf24ef1aad0687af23da144 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72217aae2cf24ef1aad0687af23da144 2024-11-27T11:37:02,934 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72217aae2cf24ef1aad0687af23da144, entries=100, sequenceid=281, filesize=9.6 K 2024-11-27T11:37:02,935 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f7a3b830a30a75170b3af25b3b7b615a in 1650ms, sequenceid=281, compaction requested=false 2024-11-27T11:37:02,938 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/recovered.edits/284.seqid, newMaxSeqId=284, maxSeqId=4 2024-11-27T11:37:02,939 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. 2024-11-27T11:37:02,939 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] regionserver.HRegion(1676): Region close journal for f7a3b830a30a75170b3af25b3b7b615a: Waiting for close lock at 1732707421285Running coprocessor pre-close hooks at 1732707421285Disabling compacts and flushes for region at 1732707421285Disabling writes for close at 1732707421285Obtaining lock to block concurrent updates at 1732707421285Preparing flush snapshotting stores in f7a3b830a30a75170b3af25b3b7b615a at 1732707421285Finished memstore snapshotting TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732707421285Flushing stores of TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a. at 1732707421286 (+1 ms)Flushing f7a3b830a30a75170b3af25b3b7b615a/A: creating writer at 1732707421286Flushing mob file f7a3b830a30a75170b3af25b3b7b615a/A: appending metadata at 1732707421290 (+4 ms)Flushing mob file f7a3b830a30a75170b3af25b3b7b615a/A: closing flushed file at 1732707421290Flushing f7a3b830a30a75170b3af25b3b7b615a/A: appending metadata at 1732707421698 (+408 ms)Flushing f7a3b830a30a75170b3af25b3b7b615a/A: closing flushed file at 1732707421698Flushing f7a3b830a30a75170b3af25b3b7b615a/B: creating writer at 1732707422105 (+407 ms)Flushing f7a3b830a30a75170b3af25b3b7b615a/B: appending metadata at 1732707422107 (+2 ms)Flushing f7a3b830a30a75170b3af25b3b7b615a/B: closing flushed file at 1732707422107Flushing f7a3b830a30a75170b3af25b3b7b615a/C: creating writer at 1732707422515 (+408 ms)Flushing f7a3b830a30a75170b3af25b3b7b615a/C: appending metadata at 1732707422517 (+2 ms)Flushing f7a3b830a30a75170b3af25b3b7b615a/C: closing flushed file at 1732707422517Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40c5de8d: reopening flushed file at 1732707422924 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@883d854: reopening flushed file at 1732707422927 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@227e51f4: reopening flushed file at 1732707422931 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for f7a3b830a30a75170b3af25b3b7b615a in 1650ms, sequenceid=281, compaction requested=false at 1732707422935 (+4 ms)Writing region close event to WAL at 1732707422936 (+1 ms)Running coprocessor post-close hooks at 1732707422939 (+3 ms)Closed at 1732707422939 2024-11-27T11:37:02,941 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=108}] handler.UnassignRegionHandler(157): Closed f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:02,941 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=f7a3b830a30a75170b3af25b3b7b615a, regionState=CLOSED 2024-11-27T11:37:02,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; CloseRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:37:02,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-27T11:37:02,944 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; CloseRegionProcedure f7a3b830a30a75170b3af25b3b7b615a, server=5b617c7edc8e,35461,1732707299005 in 1.8100 sec 2024-11-27T11:37:02,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-27T11:37:02,946 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f7a3b830a30a75170b3af25b3b7b615a, UNASSIGN in 1.8140 sec 2024-11-27T11:37:02,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-27T11:37:02,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8170 sec 2024-11-27T11:37:02,948 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707422948"}]},"ts":"1732707422948"} 2024-11-27T11:37:02,949 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T11:37:02,949 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T11:37:02,951 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8250 sec 2024-11-27T11:37:03,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-27T11:37:03,282 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:03,283 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T11:37:03,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,285 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=109, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T11:37:03,285 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=109, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,287 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,289 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/recovered.edits] 2024-11-27T11:37:03,291 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/3103f742edf446a380768898cccc0d76 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/3103f742edf446a380768898cccc0d76 2024-11-27T11:37:03,291 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/bd9fc2e501fc493d9ee216b369cef561 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/A/bd9fc2e501fc493d9ee216b369cef561 2024-11-27T11:37:03,293 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/454398b3b9464bc6b999aa4c21e171c1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/454398b3b9464bc6b999aa4c21e171c1 2024-11-27T11:37:03,294 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/57da9a309b7b403a822fcfa79a9877af to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/B/57da9a309b7b403a822fcfa79a9877af 2024-11-27T11:37:03,296 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/5242756e9c274bfc8f2d05ef902f64c2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/5242756e9c274bfc8f2d05ef902f64c2 2024-11-27T11:37:03,296 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72217aae2cf24ef1aad0687af23da144 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/C/72217aae2cf24ef1aad0687af23da144 2024-11-27T11:37:03,298 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/recovered.edits/284.seqid to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a/recovered.edits/284.seqid 2024-11-27T11:37:03,299 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,299 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T11:37:03,299 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T11:37:03,300 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T11:37:03,302 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112701cca7eb86584f54aed9be10eb7d2edb_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112701cca7eb86584f54aed9be10eb7d2edb_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,303 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271e8fe4a7099a49e9872e26073eb8611f_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411271e8fe4a7099a49e9872e26073eb8611f_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,303 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127497b7d98114a499c8a2349fcdfb93f7e_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127497b7d98114a499c8a2349fcdfb93f7e_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,304 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127596db033dd29490da78a36449ab2e332_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127596db033dd29490da78a36449ab2e332_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,305 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127774af78a0950439aa8559623792d10c6_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127774af78a0950439aa8559623792d10c6_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,306 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787fd0d7340754fa6a0a57c388b8a90a8_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112787fd0d7340754fa6a0a57c388b8a90a8_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,307 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278b61fa2f4db44cb896f9d3ab2e331cb8_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278b61fa2f4db44cb896f9d3ab2e331cb8_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,308 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278d5dad36f1d84fef984bdf24ff390d58_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411278d5dad36f1d84fef984bdf24ff390d58_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,309 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279037a36b2dc24bed98c8b4bb80260509_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279037a36b2dc24bed98c8b4bb80260509_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,309 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c3f706dd1eb743d38eddcf49bf3685ee_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127c3f706dd1eb743d38eddcf49bf3685ee_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,310 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d02955000f5548578e2596537fe619eb_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d02955000f5548578e2596537fe619eb_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,311 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d878fd6702d640a494ea429e473ca2e6_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127d878fd6702d640a494ea429e473ca2e6_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,312 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127da5209052e4d4ba895ac8755318b7d2f_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127da5209052e4d4ba895ac8755318b7d2f_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eaa066a2a0fd427c8f7014daf170c6ec_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eaa066a2a0fd427c8f7014daf170c6ec_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,313 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3555c215fc34b7ead27a292ac022824_f7a3b830a30a75170b3af25b3b7b615a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127f3555c215fc34b7ead27a292ac022824_f7a3b830a30a75170b3af25b3b7b615a 2024-11-27T11:37:03,314 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T11:37:03,316 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=109, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,317 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T11:37:03,319 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T11:37:03,320 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=109, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,320 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T11:37:03,320 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732707423320"}]},"ts":"9223372036854775807"} 2024-11-27T11:37:03,321 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T11:37:03,322 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f7a3b830a30a75170b3af25b3b7b615a, NAME => 'TestAcidGuarantees,,1732707387281.f7a3b830a30a75170b3af25b3b7b615a.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T11:37:03,322 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T11:37:03,322 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732707423322"}]},"ts":"9223372036854775807"} 2024-11-27T11:37:03,323 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T11:37:03,324 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=109, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-27T11:37:03,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-27T11:37:03,392 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T11:37:03,393 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:03,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238 (was 240), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=278 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6351 (was 6371) 2024-11-27T11:37:03,409 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=278, ProcessCount=11, AvailableMemoryMB=6351 2024-11-27T11:37:03,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:37:03,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:37:03,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:03,412 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T11:37:03,412 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:03,412 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 110 2024-11-27T11:37:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-27T11:37:03,413 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T11:37:03,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742221_1397 (size=959) 2024-11-27T11:37:03,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-27T11:37:03,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-27T11:37:03,819 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:37:03,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742222_1398 (size=53) 2024-11-27T11:37:04,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-27T11:37:04,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:37:04,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6d1385241f880cf54f36aa04a79ba3c4, disabling compactions & flushes 2024-11-27T11:37:04,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. after waiting 0 ms 2024-11-27T11:37:04,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,224 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,224 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6d1385241f880cf54f36aa04a79ba3c4: Waiting for close lock at 1732707424224Disabling compacts and flushes for region at 1732707424224Disabling writes for close at 1732707424224Writing region close event to WAL at 1732707424224Closed at 1732707424224 2024-11-27T11:37:04,225 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T11:37:04,226 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732707424225"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732707424225"}]},"ts":"1732707424225"} 2024-11-27T11:37:04,227 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T11:37:04,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T11:37:04,228 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707424228"}]},"ts":"1732707424228"} 2024-11-27T11:37:04,229 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T11:37:04,229 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, ASSIGN}] 2024-11-27T11:37:04,230 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, ASSIGN 2024-11-27T11:37:04,230 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, ASSIGN; state=OFFLINE, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=false 2024-11-27T11:37:04,381 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=6d1385241f880cf54f36aa04a79ba3c4, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:04,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, ASSIGN because future has completed 2024-11-27T11:37:04,382 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:37:04,536 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,536 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7752): Opening region: {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:37:04,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:37:04,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7794): checking encryption for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,537 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(7797): checking classloading for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,538 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,539 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d1385241f880cf54f36aa04a79ba3c4 columnFamilyName A 2024-11-27T11:37:04,539 DEBUG [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:04,540 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] regionserver.HStore(327): Store=6d1385241f880cf54f36aa04a79ba3c4/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:04,540 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,541 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d1385241f880cf54f36aa04a79ba3c4 columnFamilyName B 2024-11-27T11:37:04,541 DEBUG [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:04,541 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] regionserver.HStore(327): Store=6d1385241f880cf54f36aa04a79ba3c4/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:04,541 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,542 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6d1385241f880cf54f36aa04a79ba3c4 columnFamilyName C 2024-11-27T11:37:04,542 DEBUG [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:04,542 INFO [StoreOpener-6d1385241f880cf54f36aa04a79ba3c4-1 {}] regionserver.HStore(327): Store=6d1385241f880cf54f36aa04a79ba3c4/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:04,542 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1038): replaying wal for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,543 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,543 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,543 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1048): stopping wal replay for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,544 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1060): Cleaning up temporary data for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,544 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:37:04,545 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1093): writing seq id for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,546 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:37:04,547 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1114): Opened 6d1385241f880cf54f36aa04a79ba3c4; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61785988, jitterRate=-0.07931703329086304}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:37:04,547 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:04,547 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegion(1006): Region open journal for 6d1385241f880cf54f36aa04a79ba3c4: Running coprocessor pre-open hook at 1732707424537Writing region info on filesystem at 1732707424537Initializing all the Stores at 1732707424538 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707424538Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707424538Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707424538Cleaning up temporary data from old regions at 1732707424544 (+6 ms)Running coprocessor post-open hooks at 1732707424547 (+3 ms)Region opened successfully at 1732707424547 2024-11-27T11:37:04,548 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., pid=112, masterSystemTime=1732707424534 2024-11-27T11:37:04,549 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,549 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=112}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:04,550 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=6d1385241f880cf54f36aa04a79ba3c4, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:04,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:37:04,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-27T11:37:04,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-27T11:37:04,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; OpenRegionProcedure 6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 in 169 msec 2024-11-27T11:37:04,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-27T11:37:04,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, ASSIGN in 324 msec 2024-11-27T11:37:04,555 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T11:37:04,555 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707424555"}]},"ts":"1732707424555"} 2024-11-27T11:37:04,556 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T11:37:04,556 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=110, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T11:37:04,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-27T11:37:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-11-27T11:37:05,562 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:05,563 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ae36a78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,563 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,563 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,564 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b73b0bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,566 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28e1a48b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,566 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,567 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,568 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34602, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,568 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:37:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:37:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@332b964d 2024-11-27T11:37:05,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:37:05,570 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:37:05,571 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f759b55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,571 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,571 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,571 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c271db7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63bc87b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63ce3349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,574 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27b5f034, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600b3db6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,576 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d97070, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,576 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,576 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,576 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40398c51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,576 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,577 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29c8dbb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38ae562a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61652bcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e8e90b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e7189ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,580 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,580 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,581 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500ef9dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,582 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1098bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,583 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@370743bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,583 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,583 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,583 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbafa26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,584 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,585 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36270, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,585 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e12c476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,588 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ebab6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,588 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,588 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,588 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22fcc46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,590 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,590 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4608c021, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,591 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3395540a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,591 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,591 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,591 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be49ed3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,592 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,593 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57615c80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13bbbc70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64c74b11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36328, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,595 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@357fb77f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,596 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f2aa8d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,596 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:05,596 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:05,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:05,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35170192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:05,597 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:05,597 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:05,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23776dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:05,598 DEBUG [Thread-1806 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,598 DEBUG [Thread-1808 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,598 DEBUG [Thread-1812 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,599 DEBUG [Thread-1823 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,599 DEBUG [Thread-1814 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,599 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,599 DEBUG [Thread-1825 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,600 DEBUG [Thread-1817 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,600 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,600 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,601 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34614, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,602 DEBUG [Thread-1819 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,602 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,602 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,603 DEBUG [Thread-1821 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,604 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,604 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,605 DEBUG [Thread-1810 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:05,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:05,606 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:05,607 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34686, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:05,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2] 2024-11-27T11:37:05,611 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:05,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-27T11:37:05,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:05,613 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:05,613 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:05,614 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:05,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:05,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:05,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:05,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/30f23e353a8d4e9fb2b56a9b81d5ce90 is 50, key is test_row_0/A:col10/1732707425609/Put/seqid=0 2024-11-27T11:37:05,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742223_1399 (size=14341) 2024-11-27T11:37:05,655 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707485651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707485653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707485653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707485654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,665 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,665 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707485655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:05,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:05,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:05,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:05,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707485763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:05,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707485772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707485772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707485772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707485772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:05,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:05,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:05,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:05,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:05,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:05,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:05,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707485972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707485982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707485982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707485983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707485983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:05,990 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/30f23e353a8d4e9fb2b56a9b81d5ce90 2024-11-27T11:37:06,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c2070b5ebc294d1595331cbcadaa9961 is 50, key is test_row_0/B:col10/1732707425609/Put/seqid=0 2024-11-27T11:37:06,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742224_1400 (size=12001) 2024-11-27T11:37:06,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:06,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:06,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707486282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707486292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707486302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707486302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707486302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,307 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:06,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c2070b5ebc294d1595331cbcadaa9961 2024-11-27T11:37:06,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/d879816eae56457787bf926db4a03903 is 50, key is test_row_0/C:col10/1732707425609/Put/seqid=0 2024-11-27T11:37:06,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742225_1401 (size=12001) 2024-11-27T11:37:06,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:06,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707486792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707486802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707486813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707486813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707486822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:06,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:06,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:06,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/d879816eae56457787bf926db4a03903 2024-11-27T11:37:06,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/30f23e353a8d4e9fb2b56a9b81d5ce90 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/30f23e353a8d4e9fb2b56a9b81d5ce90 2024-11-27T11:37:06,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/30f23e353a8d4e9fb2b56a9b81d5ce90, entries=200, sequenceid=12, filesize=14.0 K 2024-11-27T11:37:06,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c2070b5ebc294d1595331cbcadaa9961 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c2070b5ebc294d1595331cbcadaa9961 2024-11-27T11:37:06,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c2070b5ebc294d1595331cbcadaa9961, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T11:37:06,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/d879816eae56457787bf926db4a03903 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/d879816eae56457787bf926db4a03903 2024-11-27T11:37:06,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/d879816eae56457787bf926db4a03903, entries=150, sequenceid=12, filesize=11.7 K 2024-11-27T11:37:06,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6d1385241f880cf54f36aa04a79ba3c4 in 1293ms, sequenceid=12, compaction requested=false 2024-11-27T11:37:06,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:06,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-27T11:37:06,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:06,990 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:37:06,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/fecdfeff876b41bc95e0c24d4e48d474 is 50, key is test_row_0/A:col10/1732707425640/Put/seqid=0 2024-11-27T11:37:06,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742226_1402 (size=12001) 2024-11-27T11:37:06,999 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/fecdfeff876b41bc95e0c24d4e48d474 2024-11-27T11:37:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/ab6ce3bc8ca14cd49e37677382a07a2c is 50, key is test_row_0/B:col10/1732707425640/Put/seqid=0 2024-11-27T11:37:07,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742227_1403 (size=12001) 2024-11-27T11:37:07,414 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/ab6ce3bc8ca14cd49e37677382a07a2c 2024-11-27T11:37:07,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/14134ebe15ef43db8f09a24cf4b17bef is 50, key is test_row_0/C:col10/1732707425640/Put/seqid=0 2024-11-27T11:37:07,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742228_1404 (size=12001) 2024-11-27T11:37:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:07,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:07,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:07,828 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/14134ebe15ef43db8f09a24cf4b17bef 2024-11-27T11:37:07,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/fecdfeff876b41bc95e0c24d4e48d474 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/fecdfeff876b41bc95e0c24d4e48d474 2024-11-27T11:37:07,836 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/fecdfeff876b41bc95e0c24d4e48d474, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T11:37:07,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/ab6ce3bc8ca14cd49e37677382a07a2c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/ab6ce3bc8ca14cd49e37677382a07a2c 2024-11-27T11:37:07,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707487828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707487831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707487833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,839 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/ab6ce3bc8ca14cd49e37677382a07a2c, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T11:37:07,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707487835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,840 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 6d1385241f880cf54f36aa04a79ba3c4 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:07,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/14134ebe15ef43db8f09a24cf4b17bef as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/14134ebe15ef43db8f09a24cf4b17bef 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,843 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/14134ebe15ef43db8f09a24cf4b17bef, entries=150, sequenceid=37, filesize=11.7 K 2024-11-27T11:37:07,844 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6d1385241f880cf54f36aa04a79ba3c4 in 854ms, sequenceid=37, compaction requested=false 2024-11-27T11:37:07,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:07,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:07,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-27T11:37:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-27T11:37:07,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:07,847 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:07,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:37:07,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-27T11:37:07,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2320 sec 2024-11-27T11:37:07,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 2.2380 sec 2024-11-27T11:37:07,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/02f36d7572d74f39b3fab1a7c8682e5f is 50, key is test_row_0/A:col10/1732707427832/Put/seqid=0 2024-11-27T11:37:07,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742229_1405 (size=12001) 2024-11-27T11:37:07,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707487955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707487956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,967 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707487957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:07,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707487961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:07,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707488072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707488072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707488072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707488072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/02f36d7572d74f39b3fab1a7c8682e5f 2024-11-27T11:37:08,264 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/6d4448294fe34e32b488dfd76aef6a4a is 50, key is test_row_0/B:col10/1732707427832/Put/seqid=0 2024-11-27T11:37:08,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742230_1406 (size=12001) 2024-11-27T11:37:08,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707488283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707488292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707488292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707488292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,457 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:37:08,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707488593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,597 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707488603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707488603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:08,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707488612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:08,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:08,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/6d4448294fe34e32b488dfd76aef6a4a 2024-11-27T11:37:08,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/71313017addd4980a372ab751d6b7e5b is 50, key is test_row_0/C:col10/1732707427832/Put/seqid=0 2024-11-27T11:37:08,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742231_1407 (size=12001) 2024-11-27T11:37:09,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/71313017addd4980a372ab751d6b7e5b 2024-11-27T11:37:09,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/02f36d7572d74f39b3fab1a7c8682e5f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/02f36d7572d74f39b3fab1a7c8682e5f 2024-11-27T11:37:09,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/02f36d7572d74f39b3fab1a7c8682e5f, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T11:37:09,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/6d4448294fe34e32b488dfd76aef6a4a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/6d4448294fe34e32b488dfd76aef6a4a 2024-11-27T11:37:09,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/6d4448294fe34e32b488dfd76aef6a4a, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T11:37:09,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/71313017addd4980a372ab751d6b7e5b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/71313017addd4980a372ab751d6b7e5b 2024-11-27T11:37:09,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/71313017addd4980a372ab751d6b7e5b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-27T11:37:09,099 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6d1385241f880cf54f36aa04a79ba3c4 in 1252ms, sequenceid=49, compaction requested=true 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:09,099 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:09,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:09,099 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:09,100 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:09,100 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:09,100 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:09,100 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:09,100 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/30f23e353a8d4e9fb2b56a9b81d5ce90, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/fecdfeff876b41bc95e0c24d4e48d474, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/02f36d7572d74f39b3fab1a7c8682e5f] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=37.4 K 2024-11-27T11:37:09,101 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:09,101 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:09,101 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c2070b5ebc294d1595331cbcadaa9961, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/ab6ce3bc8ca14cd49e37677382a07a2c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/6d4448294fe34e32b488dfd76aef6a4a] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=35.2 K 2024-11-27T11:37:09,101 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c2070b5ebc294d1595331cbcadaa9961, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732707425608 2024-11-27T11:37:09,101 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30f23e353a8d4e9fb2b56a9b81d5ce90, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732707425608 2024-11-27T11:37:09,101 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting fecdfeff876b41bc95e0c24d4e48d474, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732707425640 2024-11-27T11:37:09,101 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ab6ce3bc8ca14cd49e37677382a07a2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732707425640 2024-11-27T11:37:09,101 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02f36d7572d74f39b3fab1a7c8682e5f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732707427827 2024-11-27T11:37:09,102 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4448294fe34e32b488dfd76aef6a4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732707427827 2024-11-27T11:37:09,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:09,110 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:09,110 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:09,110 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a86990deb348487083919fba6898583e is 50, key is test_row_0/B:col10/1732707427832/Put/seqid=0 2024-11-27T11:37:09,110 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/8c280d90ee45428e9559050d0e2c1558 is 50, key is test_row_0/A:col10/1732707427832/Put/seqid=0 2024-11-27T11:37:09,111 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:09,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:37:09,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/0aa5831eac804e88a85ae5c3f2e69547 is 50, key is test_row_0/A:col10/1732707429108/Put/seqid=0 2024-11-27T11:37:09,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742233_1409 (size=12104) 2024-11-27T11:37:09,128 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/8c280d90ee45428e9559050d0e2c1558 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8c280d90ee45428e9559050d0e2c1558 2024-11-27T11:37:09,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742232_1408 (size=12104) 2024-11-27T11:37:09,133 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into 8c280d90ee45428e9559050d0e2c1558(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:09,134 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:09,134 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707429099; duration=0sec 2024-11-27T11:37:09,134 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:09,134 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:09,134 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:09,135 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:09,135 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:09,135 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:09,135 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/d879816eae56457787bf926db4a03903, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/14134ebe15ef43db8f09a24cf4b17bef, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/71313017addd4980a372ab751d6b7e5b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=35.2 K 2024-11-27T11:37:09,135 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d879816eae56457787bf926db4a03903, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732707425608 2024-11-27T11:37:09,136 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14134ebe15ef43db8f09a24cf4b17bef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732707425640 2024-11-27T11:37:09,136 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71313017addd4980a372ab751d6b7e5b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732707427827 2024-11-27T11:37:09,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742234_1410 (size=14341) 2024-11-27T11:37:09,142 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/0aa5831eac804e88a85ae5c3f2e69547 2024-11-27T11:37:09,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707489136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707489136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,152 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#340 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:09,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707489137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,153 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1d642511306a446e94d9eb5555fd5dbf is 50, key is test_row_0/C:col10/1732707427832/Put/seqid=0 2024-11-27T11:37:09,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707489139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/84aceb4b93c644d3bb636ecf4e749049 is 50, key is test_row_0/B:col10/1732707429108/Put/seqid=0 2024-11-27T11:37:09,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742235_1411 (size=12104) 2024-11-27T11:37:09,166 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1d642511306a446e94d9eb5555fd5dbf as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1d642511306a446e94d9eb5555fd5dbf 2024-11-27T11:37:09,171 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 1d642511306a446e94d9eb5555fd5dbf(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:09,172 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:09,172 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707429099; duration=0sec 2024-11-27T11:37:09,172 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:09,172 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:09,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742236_1412 (size=12001) 2024-11-27T11:37:09,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/84aceb4b93c644d3bb636ecf4e749049 2024-11-27T11:37:09,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/320de7b487f54b4980401030dffc2f71 is 50, key is test_row_0/C:col10/1732707429108/Put/seqid=0 2024-11-27T11:37:09,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742237_1413 (size=12001) 2024-11-27T11:37:09,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707489252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707489252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707489263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707489263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,271 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707489462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707489462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707489473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707489473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,535 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a86990deb348487083919fba6898583e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a86990deb348487083919fba6898583e 2024-11-27T11:37:09,539 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into a86990deb348487083919fba6898583e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:09,539 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:09,539 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707429099; duration=0sec 2024-11-27T11:37:09,539 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:09,539 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:09,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/320de7b487f54b4980401030dffc2f71 2024-11-27T11:37:09,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/0aa5831eac804e88a85ae5c3f2e69547 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0aa5831eac804e88a85ae5c3f2e69547 2024-11-27T11:37:09,599 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0aa5831eac804e88a85ae5c3f2e69547, entries=200, sequenceid=74, filesize=14.0 K 2024-11-27T11:37:09,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/84aceb4b93c644d3bb636ecf4e749049 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/84aceb4b93c644d3bb636ecf4e749049 2024-11-27T11:37:09,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/84aceb4b93c644d3bb636ecf4e749049, entries=150, sequenceid=74, filesize=11.7 K 2024-11-27T11:37:09,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/320de7b487f54b4980401030dffc2f71 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/320de7b487f54b4980401030dffc2f71 2024-11-27T11:37:09,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/320de7b487f54b4980401030dffc2f71, entries=150, sequenceid=74, filesize=11.7 K 2024-11-27T11:37:09,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6d1385241f880cf54f36aa04a79ba3c4 in 496ms, sequenceid=74, compaction requested=false 2024-11-27T11:37:09,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-27T11:37:09,772 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:09,773 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-27T11:37:09,774 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:09,775 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:09,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:09,787 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:09,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:37:09,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/891ffd1b66b7479fb36e7a037d871e18 is 50, key is test_row_0/A:col10/1732707429136/Put/seqid=0 2024-11-27T11:37:09,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742238_1414 (size=16681) 2024-11-27T11:37:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707489828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707489829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707489835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707489839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,846 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707489862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,871 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 6d1385241f880cf54f36aa04a79ba3c4 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:09,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:09,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:09,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:09,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707489952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707489952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707489952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707489952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:09,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:09,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707490172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707490172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707490172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707490173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,201 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/891ffd1b66b7479fb36e7a037d871e18 2024-11-27T11:37:10,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c709dcff7fc54b529e1aecb729e717e0 is 50, key is test_row_0/B:col10/1732707429136/Put/seqid=0 2024-11-27T11:37:10,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742239_1415 (size=12001) 2024-11-27T11:37:10,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:10,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707490492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707490492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707490492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707490492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:10,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:10,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,612 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c709dcff7fc54b529e1aecb729e717e0 2024-11-27T11:37:10,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/5bd12fa4fa734f5a8efc1a298c2f7bf7 is 50, key is test_row_0/C:col10/1732707429136/Put/seqid=0 2024-11-27T11:37:10,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742240_1416 (size=12001) 2024-11-27T11:37:10,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:10,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:10,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:10,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:10,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:11,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:11,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:11,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707491002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:11,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707491012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:11,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707491013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:11,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:11,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707491013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:11,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:11,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/5bd12fa4fa734f5a8efc1a298c2f7bf7 2024-11-27T11:37:11,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/891ffd1b66b7479fb36e7a037d871e18 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/891ffd1b66b7479fb36e7a037d871e18 2024-11-27T11:37:11,030 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/891ffd1b66b7479fb36e7a037d871e18, entries=250, sequenceid=90, filesize=16.3 K 2024-11-27T11:37:11,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c709dcff7fc54b529e1aecb729e717e0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c709dcff7fc54b529e1aecb729e717e0 2024-11-27T11:37:11,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c709dcff7fc54b529e1aecb729e717e0, entries=150, sequenceid=90, filesize=11.7 K 2024-11-27T11:37:11,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/5bd12fa4fa734f5a8efc1a298c2f7bf7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/5bd12fa4fa734f5a8efc1a298c2f7bf7 2024-11-27T11:37:11,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/5bd12fa4fa734f5a8efc1a298c2f7bf7, entries=150, sequenceid=90, filesize=11.7 K 2024-11-27T11:37:11,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6d1385241f880cf54f36aa04a79ba3c4 in 1254ms, sequenceid=90, compaction requested=true 2024-11-27T11:37:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:11,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:11,042 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:11,042 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:11,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43126 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:11,043 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:11,043 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:11,043 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a86990deb348487083919fba6898583e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/84aceb4b93c644d3bb636ecf4e749049, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c709dcff7fc54b529e1aecb729e717e0] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=35.3 K 2024-11-27T11:37:11,043 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8c280d90ee45428e9559050d0e2c1558, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0aa5831eac804e88a85ae5c3f2e69547, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/891ffd1b66b7479fb36e7a037d871e18] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=42.1 K 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a86990deb348487083919fba6898583e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732707427827 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c280d90ee45428e9559050d0e2c1558, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732707427827 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 84aceb4b93c644d3bb636ecf4e749049, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732707427954 2024-11-27T11:37:11,043 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0aa5831eac804e88a85ae5c3f2e69547, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732707427954 2024-11-27T11:37:11,044 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c709dcff7fc54b529e1aecb729e717e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732707429133 2024-11-27T11:37:11,044 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 891ffd1b66b7479fb36e7a037d871e18, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732707429133 2024-11-27T11:37:11,050 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#346 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:11,051 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a9ccd3522886400aa918ed443fef8121 is 50, key is test_row_0/B:col10/1732707429136/Put/seqid=0 2024-11-27T11:37:11,053 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#347 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:11,054 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/aee2a513e4ac4edb92254c9aab5e15fb is 50, key is test_row_0/A:col10/1732707429136/Put/seqid=0 2024-11-27T11:37:11,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742242_1418 (size=12207) 2024-11-27T11:37:11,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742241_1417 (size=12207) 2024-11-27T11:37:11,063 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a9ccd3522886400aa918ed443fef8121 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9ccd3522886400aa918ed443fef8121 2024-11-27T11:37:11,067 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into a9ccd3522886400aa918ed443fef8121(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:11,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:11,067 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707431041; duration=0sec 2024-11-27T11:37:11,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:11,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:11,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:11,068 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:11,068 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:11,068 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:11,068 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1d642511306a446e94d9eb5555fd5dbf, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/320de7b487f54b4980401030dffc2f71, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/5bd12fa4fa734f5a8efc1a298c2f7bf7] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=35.3 K 2024-11-27T11:37:11,068 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d642511306a446e94d9eb5555fd5dbf, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1732707427827 2024-11-27T11:37:11,069 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 320de7b487f54b4980401030dffc2f71, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732707427954 2024-11-27T11:37:11,069 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd12fa4fa734f5a8efc1a298c2f7bf7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732707429133 2024-11-27T11:37:11,076 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#348 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:11,077 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/642b2a3d53d14b7986a955b56bf627d5 is 50, key is test_row_0/C:col10/1732707429136/Put/seqid=0 2024-11-27T11:37:11,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742243_1419 (size=12207) 2024-11-27T11:37:11,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-27T11:37:11,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:11,153 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:37:11,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/06564325a7cb40dd90cefdeaa6d069da is 50, key is test_row_0/A:col10/1732707429827/Put/seqid=0 2024-11-27T11:37:11,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742244_1420 (size=12001) 2024-11-27T11:37:11,464 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/aee2a513e4ac4edb92254c9aab5e15fb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/aee2a513e4ac4edb92254c9aab5e15fb 2024-11-27T11:37:11,468 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into aee2a513e4ac4edb92254c9aab5e15fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:11,468 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:11,468 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707431041; duration=0sec 2024-11-27T11:37:11,468 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:11,468 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:11,485 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/642b2a3d53d14b7986a955b56bf627d5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/642b2a3d53d14b7986a955b56bf627d5 2024-11-27T11:37:11,489 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 642b2a3d53d14b7986a955b56bf627d5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:11,489 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:11,489 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707431042; duration=0sec 2024-11-27T11:37:11,489 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:11,489 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:11,561 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/06564325a7cb40dd90cefdeaa6d069da 2024-11-27T11:37:11,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/3b759618a17f4795b4414e4828ac146e is 50, key is test_row_0/B:col10/1732707429827/Put/seqid=0 2024-11-27T11:37:11,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742245_1421 (size=12001) 2024-11-27T11:37:11,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:11,972 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/3b759618a17f4795b4414e4828ac146e 2024-11-27T11:37:11,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/efb2760f92894c4c84e214ffa66735ff is 50, key is test_row_0/C:col10/1732707429827/Put/seqid=0 2024-11-27T11:37:11,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742246_1422 (size=12001) 2024-11-27T11:37:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:12,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:12,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707492050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707492052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707492052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,061 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707492052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707492162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707492173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707492173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707492173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,383 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/efb2760f92894c4c84e214ffa66735ff 2024-11-27T11:37:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/06564325a7cb40dd90cefdeaa6d069da as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/06564325a7cb40dd90cefdeaa6d069da 2024-11-27T11:37:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707492383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707492383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,390 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/06564325a7cb40dd90cefdeaa6d069da, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T11:37:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707492383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707492383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/3b759618a17f4795b4414e4828ac146e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/3b759618a17f4795b4414e4828ac146e 2024-11-27T11:37:12,394 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/3b759618a17f4795b4414e4828ac146e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T11:37:12,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/efb2760f92894c4c84e214ffa66735ff as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/efb2760f92894c4c84e214ffa66735ff 2024-11-27T11:37:12,397 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/efb2760f92894c4c84e214ffa66735ff, entries=150, sequenceid=114, filesize=11.7 K 2024-11-27T11:37:12,398 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6d1385241f880cf54f36aa04a79ba3c4 in 1245ms, sequenceid=114, compaction requested=false 2024-11-27T11:37:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:12,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-27T11:37:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-27T11:37:12,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-27T11:37:12,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6240 sec 2024-11-27T11:37:12,403 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.6290 sec 2024-11-27T11:37:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:12,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:12,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T11:37:12,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/40d8b173644042aa9c32c4db8cf4a4af is 50, key is test_row_0/A:col10/1732707432697/Put/seqid=0 2024-11-27T11:37:12,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742247_1423 (size=14441) 2024-11-27T11:37:12,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/40d8b173644042aa9c32c4db8cf4a4af 2024-11-27T11:37:12,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a2f8a76f4dd94fc490c50b06ab174f2e is 50, key is test_row_0/B:col10/1732707432697/Put/seqid=0 2024-11-27T11:37:12,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742248_1424 (size=12051) 2024-11-27T11:37:12,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a2f8a76f4dd94fc490c50b06ab174f2e 2024-11-27T11:37:12,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1679b05c5df24847b724d271d3091924 is 50, key is test_row_0/C:col10/1732707432697/Put/seqid=0 2024-11-27T11:37:12,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742249_1425 (size=12051) 2024-11-27T11:37:12,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707492739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,751 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707492746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707492750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707492751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707492853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,861 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707492862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707492862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707492862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:12,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707493073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,079 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707493082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707493082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707493082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1679b05c5df24847b724d271d3091924 2024-11-27T11:37:13,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/40d8b173644042aa9c32c4db8cf4a4af as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/40d8b173644042aa9c32c4db8cf4a4af 2024-11-27T11:37:13,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/40d8b173644042aa9c32c4db8cf4a4af, entries=200, sequenceid=131, filesize=14.1 K 2024-11-27T11:37:13,155 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a2f8a76f4dd94fc490c50b06ab174f2e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a2f8a76f4dd94fc490c50b06ab174f2e 2024-11-27T11:37:13,158 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a2f8a76f4dd94fc490c50b06ab174f2e, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T11:37:13,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1679b05c5df24847b724d271d3091924 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1679b05c5df24847b724d271d3091924 2024-11-27T11:37:13,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1679b05c5df24847b724d271d3091924, entries=150, sequenceid=131, filesize=11.8 K 2024-11-27T11:37:13,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6d1385241f880cf54f36aa04a79ba3c4 in 465ms, sequenceid=131, compaction requested=true 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:13,164 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:13,164 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38649 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:13,165 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:13,165 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:13,165 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9ccd3522886400aa918ed443fef8121, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/3b759618a17f4795b4414e4828ac146e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a2f8a76f4dd94fc490c50b06ab174f2e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=35.4 K 2024-11-27T11:37:13,165 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/aee2a513e4ac4edb92254c9aab5e15fb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/06564325a7cb40dd90cefdeaa6d069da, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/40d8b173644042aa9c32c4db8cf4a4af] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=37.7 K 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ccd3522886400aa918ed443fef8121, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732707429133 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting aee2a513e4ac4edb92254c9aab5e15fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732707429133 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06564325a7cb40dd90cefdeaa6d069da, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732707429827 2024-11-27T11:37:13,165 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b759618a17f4795b4414e4828ac146e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732707429827 2024-11-27T11:37:13,166 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40d8b173644042aa9c32c4db8cf4a4af, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732707432035 2024-11-27T11:37:13,166 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f8a76f4dd94fc490c50b06ab174f2e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732707432035 2024-11-27T11:37:13,173 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#355 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:13,174 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:13,174 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/b0fad81de1e8404c9d12bdceec06efee is 50, key is test_row_0/A:col10/1732707432697/Put/seqid=0 2024-11-27T11:37:13,174 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/1d3467e40b2244b1ab405a2186f2dd44 is 50, key is test_row_0/B:col10/1732707432697/Put/seqid=0 2024-11-27T11:37:13,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742250_1426 (size=12359) 2024-11-27T11:37:13,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742251_1427 (size=12359) 2024-11-27T11:37:13,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:13,390 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:13,391 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-27T11:37:13,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/212a8afe8b834734a7e80045005e2a68 is 50, key is test_row_0/A:col10/1732707433389/Put/seqid=0 2024-11-27T11:37:13,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742252_1428 (size=14541) 2024-11-27T11:37:13,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707493426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707493427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707493428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707493428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707493532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707493543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707493543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707493543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,594 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/b0fad81de1e8404c9d12bdceec06efee as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b0fad81de1e8404c9d12bdceec06efee 2024-11-27T11:37:13,598 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into b0fad81de1e8404c9d12bdceec06efee(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:13,599 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:13,599 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707433164; duration=0sec 2024-11-27T11:37:13,599 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:13,599 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:13,599 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:13,600 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:13,600 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:13,600 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:13,600 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/642b2a3d53d14b7986a955b56bf627d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/efb2760f92894c4c84e214ffa66735ff, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1679b05c5df24847b724d271d3091924] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=35.4 K 2024-11-27T11:37:13,600 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642b2a3d53d14b7986a955b56bf627d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732707429133 2024-11-27T11:37:13,601 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting efb2760f92894c4c84e214ffa66735ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732707429827 2024-11-27T11:37:13,601 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1679b05c5df24847b724d271d3091924, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732707432035 2024-11-27T11:37:13,602 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/1d3467e40b2244b1ab405a2186f2dd44 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/1d3467e40b2244b1ab405a2186f2dd44 2024-11-27T11:37:13,606 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into 1d3467e40b2244b1ab405a2186f2dd44(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:13,606 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:13,606 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707433164; duration=0sec 2024-11-27T11:37:13,606 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:13,606 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:13,607 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:13,607 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/47886ce158ef40d19942ac13f6b3c077 is 50, key is test_row_0/C:col10/1732707432697/Put/seqid=0 2024-11-27T11:37:13,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742253_1429 (size=12359) 2024-11-27T11:37:13,616 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/47886ce158ef40d19942ac13f6b3c077 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/47886ce158ef40d19942ac13f6b3c077 2024-11-27T11:37:13,620 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 47886ce158ef40d19942ac13f6b3c077(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:13,620 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:13,620 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707433164; duration=0sec 2024-11-27T11:37:13,620 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:13,620 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:13,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707493752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707493763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707493763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707493763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/212a8afe8b834734a7e80045005e2a68 2024-11-27T11:37:13,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/88ce04880fba4fb283be675fdda0b780 is 50, key is test_row_0/B:col10/1732707433389/Put/seqid=0 2024-11-27T11:37:13,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742254_1430 (size=12151) 2024-11-27T11:37:13,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707493902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:13,909 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 6d1385241f880cf54f36aa04a79ba3c4 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:13,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-27T11:37:13,943 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:13,943 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:13,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-27T11:37:13,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T11:37:13,945 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:13,945 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:13,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:14,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T11:37:14,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707494062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,071 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707494073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707494073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707494073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T11:37:14,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:14,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,098 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/88ce04880fba4fb283be675fdda0b780 2024-11-27T11:37:14,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/f8de38d28c4b4f3fa39d441da48c1ee4 is 50, key is test_row_0/C:col10/1732707433389/Put/seqid=0 2024-11-27T11:37:14,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742255_1431 (size=12151) 2024-11-27T11:37:14,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T11:37:14,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:14,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T11:37:14,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T11:37:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T11:37:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:14,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T11:37:14,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707494583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707494583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707494583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707494583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:14,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:14,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/f8de38d28c4b4f3fa39d441da48c1ee4 2024-11-27T11:37:14,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/212a8afe8b834734a7e80045005e2a68 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/212a8afe8b834734a7e80045005e2a68 2024-11-27T11:37:14,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/212a8afe8b834734a7e80045005e2a68, entries=200, sequenceid=153, filesize=14.2 K 2024-11-27T11:37:14,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/88ce04880fba4fb283be675fdda0b780 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/88ce04880fba4fb283be675fdda0b780 2024-11-27T11:37:14,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/88ce04880fba4fb283be675fdda0b780, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T11:37:14,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/f8de38d28c4b4f3fa39d441da48c1ee4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f8de38d28c4b4f3fa39d441da48c1ee4 2024-11-27T11:37:14,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f8de38d28c4b4f3fa39d441da48c1ee4, entries=150, sequenceid=153, filesize=11.9 K 2024-11-27T11:37:14,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 6d1385241f880cf54f36aa04a79ba3c4 in 1253ms, sequenceid=153, compaction requested=false 2024-11-27T11:37:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:14,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-27T11:37:14,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:14,709 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-27T11:37:14,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/46f1af0ad0c344b4bc9adf8464543deb is 50, key is test_row_0/A:col10/1732707433413/Put/seqid=0 2024-11-27T11:37:14,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742256_1432 (size=12151) 2024-11-27T11:37:14,721 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/46f1af0ad0c344b4bc9adf8464543deb 2024-11-27T11:37:14,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/818ee73c185440a6b3516ef32551003c is 50, key is test_row_0/B:col10/1732707433413/Put/seqid=0 2024-11-27T11:37:14,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742257_1433 (size=12151) 2024-11-27T11:37:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T11:37:15,130 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/818ee73c185440a6b3516ef32551003c 2024-11-27T11:37:15,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/164ce1259b4d47a0afde43143f122517 is 50, key is test_row_0/C:col10/1732707433413/Put/seqid=0 2024-11-27T11:37:15,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742258_1434 (size=12151) 2024-11-27T11:37:15,546 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/164ce1259b4d47a0afde43143f122517 2024-11-27T11:37:15,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/46f1af0ad0c344b4bc9adf8464543deb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/46f1af0ad0c344b4bc9adf8464543deb 2024-11-27T11:37:15,553 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/46f1af0ad0c344b4bc9adf8464543deb, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T11:37:15,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/818ee73c185440a6b3516ef32551003c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/818ee73c185440a6b3516ef32551003c 2024-11-27T11:37:15,558 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/818ee73c185440a6b3516ef32551003c, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T11:37:15,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/164ce1259b4d47a0afde43143f122517 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/164ce1259b4d47a0afde43143f122517 2024-11-27T11:37:15,566 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/164ce1259b4d47a0afde43143f122517, entries=150, sequenceid=171, filesize=11.9 K 2024-11-27T11:37:15,567 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=0 B/0 for 6d1385241f880cf54f36aa04a79ba3c4 in 857ms, sequenceid=171, compaction requested=true 2024-11-27T11:37:15,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:15,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:15,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-27T11:37:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-27T11:37:15,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-27T11:37:15,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6230 sec 2024-11-27T11:37:15,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.6280 sec 2024-11-27T11:37:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:15,607 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:15,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:15,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/4c36aaf8e2394cfb9069513525d7a873 is 50, key is test_row_0/A:col10/1732707435605/Put/seqid=0 2024-11-27T11:37:15,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742259_1435 (size=14541) 2024-11-27T11:37:15,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707495655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,667 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707495655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707495656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707495667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707495772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707495772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707495772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,777 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707495782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707495982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707495982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,988 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707495982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707496002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/4c36aaf8e2394cfb9069513525d7a873 2024-11-27T11:37:16,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/4b7c3965fadf490db1cdf0a502e1702d is 50, key is test_row_0/B:col10/1732707435605/Put/seqid=0 2024-11-27T11:37:16,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742260_1436 (size=12151) 2024-11-27T11:37:16,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-27T11:37:16,093 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:16,093 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:16,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-27T11:37:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T11:37:16,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:16,095 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:16,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T11:37:16,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-27T11:37:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707496292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707496292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707496292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707496322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,330 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-27T11:37:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T11:37:16,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/4b7c3965fadf490db1cdf0a502e1702d 2024-11-27T11:37:16,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/79d197d435724a4f9d80095fc36d3f7e is 50, key is test_row_0/C:col10/1732707435605/Put/seqid=0 2024-11-27T11:37:16,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742261_1437 (size=12151) 2024-11-27T11:37:16,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-27T11:37:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-27T11:37:16,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:16,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T11:37:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707496802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707496802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707496802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/79d197d435724a4f9d80095fc36d3f7e 2024-11-27T11:37:16,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/4c36aaf8e2394cfb9069513525d7a873 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/4c36aaf8e2394cfb9069513525d7a873 2024-11-27T11:37:16,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/4c36aaf8e2394cfb9069513525d7a873, entries=200, sequenceid=183, filesize=14.2 K 2024-11-27T11:37:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/4b7c3965fadf490db1cdf0a502e1702d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/4b7c3965fadf490db1cdf0a502e1702d 2024-11-27T11:37:16,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:16,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707496843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:16,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:16,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/4b7c3965fadf490db1cdf0a502e1702d, entries=150, sequenceid=183, filesize=11.9 K 2024-11-27T11:37:16,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/79d197d435724a4f9d80095fc36d3f7e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/79d197d435724a4f9d80095fc36d3f7e 2024-11-27T11:37:16,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/79d197d435724a4f9d80095fc36d3f7e, entries=150, sequenceid=183, filesize=11.9 K 2024-11-27T11:37:16,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6d1385241f880cf54f36aa04a79ba3c4 in 1247ms, sequenceid=183, compaction requested=true 2024-11-27T11:37:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:16,854 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:37:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:16,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:16,855 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:37:16,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:16,855 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53592 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:37:16,856 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:16,856 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,856 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b0fad81de1e8404c9d12bdceec06efee, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/212a8afe8b834734a7e80045005e2a68, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/46f1af0ad0c344b4bc9adf8464543deb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/4c36aaf8e2394cfb9069513525d7a873] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=52.3 K 2024-11-27T11:37:16,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:37:16,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:16,856 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,856 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/1d3467e40b2244b1ab405a2186f2dd44, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/88ce04880fba4fb283be675fdda0b780, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/818ee73c185440a6b3516ef32551003c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/4b7c3965fadf490db1cdf0a502e1702d] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=47.7 K 2024-11-27T11:37:16,856 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0fad81de1e8404c9d12bdceec06efee, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732707432035 2024-11-27T11:37:16,856 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d3467e40b2244b1ab405a2186f2dd44, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732707432035 2024-11-27T11:37:16,857 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 212a8afe8b834734a7e80045005e2a68, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732707432724 2024-11-27T11:37:16,857 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 88ce04880fba4fb283be675fdda0b780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732707432724 2024-11-27T11:37:16,857 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f1af0ad0c344b4bc9adf8464543deb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732707433410 2024-11-27T11:37:16,857 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 818ee73c185440a6b3516ef32551003c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732707433410 2024-11-27T11:37:16,857 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c36aaf8e2394cfb9069513525d7a873, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732707435604 2024-11-27T11:37:16,857 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b7c3965fadf490db1cdf0a502e1702d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732707435605 2024-11-27T11:37:16,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-27T11:37:16,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:16,859 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:37:16,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/370a5da61bad44e8a6e60e8cc468b2a1 is 50, key is test_row_0/A:col10/1732707435655/Put/seqid=0 2024-11-27T11:37:16,871 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:16,871 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/5bb483a8714c422e9c4c946dbcdd93a7 is 50, key is test_row_0/B:col10/1732707435605/Put/seqid=0 2024-11-27T11:37:16,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742262_1438 (size=12151) 2024-11-27T11:37:16,872 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/370a5da61bad44e8a6e60e8cc468b2a1 2024-11-27T11:37:16,883 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:16,883 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/64db0b231fcb440995978264922349ad is 50, key is test_row_0/A:col10/1732707435605/Put/seqid=0 2024-11-27T11:37:16,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/dcc3994ac1df4bffbc4d7afa85cdf8c9 is 50, key is test_row_0/B:col10/1732707435655/Put/seqid=0 2024-11-27T11:37:16,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742264_1440 (size=12595) 2024-11-27T11:37:16,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742263_1439 (size=12595) 2024-11-27T11:37:16,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742265_1441 (size=12151) 2024-11-27T11:37:16,903 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/dcc3994ac1df4bffbc4d7afa85cdf8c9 2024-11-27T11:37:16,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/97b92ddd025042c3afc3d7f1ee0f70bf is 50, key is test_row_0/C:col10/1732707435655/Put/seqid=0 2024-11-27T11:37:16,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742266_1442 (size=12151) 2024-11-27T11:37:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T11:37:17,298 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/64db0b231fcb440995978264922349ad as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/64db0b231fcb440995978264922349ad 2024-11-27T11:37:17,303 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into 64db0b231fcb440995978264922349ad(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:17,303 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:17,303 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=12, startTime=1732707436854; duration=0sec 2024-11-27T11:37:17,303 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:17,303 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:17,303 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-27T11:37:17,303 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/5bb483a8714c422e9c4c946dbcdd93a7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/5bb483a8714c422e9c4c946dbcdd93a7 2024-11-27T11:37:17,304 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-27T11:37:17,304 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:17,304 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:17,304 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/47886ce158ef40d19942ac13f6b3c077, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f8de38d28c4b4f3fa39d441da48c1ee4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/164ce1259b4d47a0afde43143f122517, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/79d197d435724a4f9d80095fc36d3f7e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=47.7 K 2024-11-27T11:37:17,304 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47886ce158ef40d19942ac13f6b3c077, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1732707432035 2024-11-27T11:37:17,305 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8de38d28c4b4f3fa39d441da48c1ee4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1732707432724 2024-11-27T11:37:17,305 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 164ce1259b4d47a0afde43143f122517, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732707433410 2024-11-27T11:37:17,305 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79d197d435724a4f9d80095fc36d3f7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732707435605 2024-11-27T11:37:17,308 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into 5bb483a8714c422e9c4c946dbcdd93a7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:17,308 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:17,308 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=12, startTime=1732707436854; duration=0sec 2024-11-27T11:37:17,308 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:17,308 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:17,313 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:17,313 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/2655126152174d23b5b21e2c8dd5b993 is 50, key is test_row_0/C:col10/1732707435605/Put/seqid=0 2024-11-27T11:37:17,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742267_1443 (size=12595) 2024-11-27T11:37:17,318 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/97b92ddd025042c3afc3d7f1ee0f70bf 2024-11-27T11:37:17,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/370a5da61bad44e8a6e60e8cc468b2a1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/370a5da61bad44e8a6e60e8cc468b2a1 2024-11-27T11:37:17,332 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/370a5da61bad44e8a6e60e8cc468b2a1, entries=150, sequenceid=207, filesize=11.9 K 2024-11-27T11:37:17,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/dcc3994ac1df4bffbc4d7afa85cdf8c9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/dcc3994ac1df4bffbc4d7afa85cdf8c9 2024-11-27T11:37:17,339 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/dcc3994ac1df4bffbc4d7afa85cdf8c9, entries=150, sequenceid=207, filesize=11.9 K 2024-11-27T11:37:17,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/97b92ddd025042c3afc3d7f1ee0f70bf as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/97b92ddd025042c3afc3d7f1ee0f70bf 2024-11-27T11:37:17,344 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/97b92ddd025042c3afc3d7f1ee0f70bf, entries=150, sequenceid=207, filesize=11.9 K 2024-11-27T11:37:17,345 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 6d1385241f880cf54f36aa04a79ba3c4 in 486ms, sequenceid=207, compaction requested=false 2024-11-27T11:37:17,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:17,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:17,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-27T11:37:17,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-27T11:37:17,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-27T11:37:17,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2510 sec 2024-11-27T11:37:17,351 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.2570 sec 2024-11-27T11:37:17,720 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/2655126152174d23b5b21e2c8dd5b993 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/2655126152174d23b5b21e2c8dd5b993 2024-11-27T11:37:17,736 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 2655126152174d23b5b21e2c8dd5b993(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:17,736 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:17,736 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=12, startTime=1732707436854; duration=0sec 2024-11-27T11:37:17,736 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:17,736 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:17,832 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:17,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:17,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/404fc64d597b4d0bae61fe0646643cf6 is 50, key is test_row_0/A:col10/1732707437830/Put/seqid=0 2024-11-27T11:37:17,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742268_1444 (size=14541) 2024-11-27T11:37:17,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707497903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:17,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:17,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:17,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707497916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:17,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:17,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707497918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:17,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707498022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707498022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707498032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707498042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707498233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/404fc64d597b4d0bae61fe0646643cf6 2024-11-27T11:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707498233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-27T11:37:18,243 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:18,243 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:18,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-27T11:37:18,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T11:37:18,245 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:18,245 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:18,246 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:18,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b5b7aaf8a61545de9cf2744db83eab49 is 50, key is test_row_0/B:col10/1732707437830/Put/seqid=0 2024-11-27T11:37:18,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707498242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,251 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742269_1445 (size=12151) 2024-11-27T11:37:18,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b5b7aaf8a61545de9cf2744db83eab49 2024-11-27T11:37:18,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707498253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,260 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/ead93a979a234877b29703279c2b439f is 50, key is test_row_0/C:col10/1732707437830/Put/seqid=0 2024-11-27T11:37:18,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742270_1446 (size=12151) 2024-11-27T11:37:18,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T11:37:18,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T11:37:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:18,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:18,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707498543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T11:37:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:18,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:18,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:18,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707498552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T11:37:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707498562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707498562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:18,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:18,663 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=221 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/ead93a979a234877b29703279c2b439f 2024-11-27T11:37:18,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/404fc64d597b4d0bae61fe0646643cf6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/404fc64d597b4d0bae61fe0646643cf6 2024-11-27T11:37:18,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/404fc64d597b4d0bae61fe0646643cf6, entries=200, sequenceid=221, filesize=14.2 K 2024-11-27T11:37:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b5b7aaf8a61545de9cf2744db83eab49 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b5b7aaf8a61545de9cf2744db83eab49 2024-11-27T11:37:18,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b5b7aaf8a61545de9cf2744db83eab49, entries=150, sequenceid=221, filesize=11.9 K 2024-11-27T11:37:18,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/ead93a979a234877b29703279c2b439f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ead93a979a234877b29703279c2b439f 2024-11-27T11:37:18,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ead93a979a234877b29703279c2b439f, entries=150, sequenceid=221, filesize=11.9 K 2024-11-27T11:37:18,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6d1385241f880cf54f36aa04a79ba3c4 in 846ms, sequenceid=221, compaction requested=true 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:18,679 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:18,679 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:18,680 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:18,680 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:18,680 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:18,680 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:18,680 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,680 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,680 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/5bb483a8714c422e9c4c946dbcdd93a7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/dcc3994ac1df4bffbc4d7afa85cdf8c9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b5b7aaf8a61545de9cf2744db83eab49] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.0 K 2024-11-27T11:37:18,680 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/64db0b231fcb440995978264922349ad, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/370a5da61bad44e8a6e60e8cc468b2a1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/404fc64d597b4d0bae61fe0646643cf6] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=38.4 K 2024-11-27T11:37:18,680 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bb483a8714c422e9c4c946dbcdd93a7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732707435605 2024-11-27T11:37:18,680 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64db0b231fcb440995978264922349ad, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732707435605 2024-11-27T11:37:18,681 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 370a5da61bad44e8a6e60e8cc468b2a1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732707435636 2024-11-27T11:37:18,681 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc3994ac1df4bffbc4d7afa85cdf8c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732707435636 2024-11-27T11:37:18,681 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 404fc64d597b4d0bae61fe0646643cf6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732707437826 2024-11-27T11:37:18,681 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b7aaf8a61545de9cf2744db83eab49, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732707437826 2024-11-27T11:37:18,687 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:18,687 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/8d2c0776cc454b5cafa26bac6d2aee3b is 50, key is test_row_0/A:col10/1732707437830/Put/seqid=0 2024-11-27T11:37:18,696 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:18,696 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b917a2ada59f46acb94f8de9939b8e2c is 50, key is test_row_0/B:col10/1732707437830/Put/seqid=0 2024-11-27T11:37:18,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742271_1447 (size=12697) 2024-11-27T11:37:18,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-27T11:37:18,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:18,704 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:37:18,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742272_1448 (size=12697) 2024-11-27T11:37:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/b1b134cde87840be9cb00ce39fa0f96d is 50, key is test_row_0/A:col10/1732707437916/Put/seqid=0 2024-11-27T11:37:18,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742273_1449 (size=12151) 2024-11-27T11:37:18,718 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/b1b134cde87840be9cb00ce39fa0f96d 2024-11-27T11:37:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/77ef391a2ea64f5189766b9c4a41d0d1 is 50, key is test_row_0/B:col10/1732707437916/Put/seqid=0 2024-11-27T11:37:18,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742274_1450 (size=12151) 2024-11-27T11:37:18,732 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/77ef391a2ea64f5189766b9c4a41d0d1 2024-11-27T11:37:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/a7d223660b5940c2ba6b2657cbfd5529 is 50, key is test_row_0/C:col10/1732707437916/Put/seqid=0 2024-11-27T11:37:18,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742275_1451 (size=12151) 2024-11-27T11:37:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T11:37:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:19,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:19,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707499085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707499087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707499087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707499088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,104 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/8d2c0776cc454b5cafa26bac6d2aee3b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8d2c0776cc454b5cafa26bac6d2aee3b 2024-11-27T11:37:19,108 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into 8d2c0776cc454b5cafa26bac6d2aee3b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:19,108 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:19,108 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707438679; duration=0sec 2024-11-27T11:37:19,108 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:19,108 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:19,108 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:19,109 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:19,109 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:19,109 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,109 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/2655126152174d23b5b21e2c8dd5b993, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/97b92ddd025042c3afc3d7f1ee0f70bf, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ead93a979a234877b29703279c2b439f] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.0 K 2024-11-27T11:37:19,110 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2655126152174d23b5b21e2c8dd5b993, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1732707435605 2024-11-27T11:37:19,110 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97b92ddd025042c3afc3d7f1ee0f70bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732707435636 2024-11-27T11:37:19,110 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b917a2ada59f46acb94f8de9939b8e2c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b917a2ada59f46acb94f8de9939b8e2c 2024-11-27T11:37:19,111 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ead93a979a234877b29703279c2b439f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732707437826 2024-11-27T11:37:19,115 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into b917a2ada59f46acb94f8de9939b8e2c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:19,115 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:19,115 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707438679; duration=0sec 2024-11-27T11:37:19,115 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:19,115 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:19,117 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:19,117 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/057f877bb4274e7fa501e267713712d9 is 50, key is test_row_0/C:col10/1732707437830/Put/seqid=0 2024-11-27T11:37:19,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742276_1452 (size=12697) 2024-11-27T11:37:19,125 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/057f877bb4274e7fa501e267713712d9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/057f877bb4274e7fa501e267713712d9 2024-11-27T11:37:19,129 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 057f877bb4274e7fa501e267713712d9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:19,129 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:19,129 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707438679; duration=0sec 2024-11-27T11:37:19,129 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:19,129 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:19,143 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/a7d223660b5940c2ba6b2657cbfd5529 2024-11-27T11:37:19,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/b1b134cde87840be9cb00ce39fa0f96d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b1b134cde87840be9cb00ce39fa0f96d 2024-11-27T11:37:19,149 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b1b134cde87840be9cb00ce39fa0f96d, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T11:37:19,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/77ef391a2ea64f5189766b9c4a41d0d1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/77ef391a2ea64f5189766b9c4a41d0d1 2024-11-27T11:37:19,153 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/77ef391a2ea64f5189766b9c4a41d0d1, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T11:37:19,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/a7d223660b5940c2ba6b2657cbfd5529 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a7d223660b5940c2ba6b2657cbfd5529 2024-11-27T11:37:19,157 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a7d223660b5940c2ba6b2657cbfd5529, entries=150, sequenceid=246, filesize=11.9 K 2024-11-27T11:37:19,158 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6d1385241f880cf54f36aa04a79ba3c4 in 454ms, sequenceid=246, compaction requested=false 2024-11-27T11:37:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-27T11:37:19,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-27T11:37:19,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-27T11:37:19,161 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 914 msec 2024-11-27T11:37:19,163 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 918 msec 2024-11-27T11:37:19,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:19,195 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:19,195 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:37:19,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/0d9d6b86e82c4e7e97913961bfe565eb is 50, key is test_row_0/A:col10/1732707439084/Put/seqid=0 2024-11-27T11:37:19,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742277_1453 (size=12197) 2024-11-27T11:37:19,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707499239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707499240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707499241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707499246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707499352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707499352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707499352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707499362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-27T11:37:19,383 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:19,383 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-27T11:37:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T11:37:19,385 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:19,386 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:19,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T11:37:19,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707499562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707499562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707499562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707499583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/0d9d6b86e82c4e7e97913961bfe565eb 2024-11-27T11:37:19,613 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/f6b49a6bacb44c6282e5bf970cf80911 is 50, key is test_row_0/B:col10/1732707439084/Put/seqid=0 2024-11-27T11:37:19,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742278_1454 (size=9807) 2024-11-27T11:37:19,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T11:37:19,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707499872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707499872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707499872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707499903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:19,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:19,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:19,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T11:37:20,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/f6b49a6bacb44c6282e5bf970cf80911 2024-11-27T11:37:20,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/37b0aa98dc6e4d198edc1bfc78998b19 is 50, key is test_row_0/C:col10/1732707439084/Put/seqid=0 2024-11-27T11:37:20,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742279_1455 (size=9807) 2024-11-27T11:37:20,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:20,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:20,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707500382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:20,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707500382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:20,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:20,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707500393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707500413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:20,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:20,430 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/37b0aa98dc6e4d198edc1bfc78998b19 2024-11-27T11:37:20,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/0d9d6b86e82c4e7e97913961bfe565eb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0d9d6b86e82c4e7e97913961bfe565eb 2024-11-27T11:37:20,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0d9d6b86e82c4e7e97913961bfe565eb, entries=150, sequenceid=261, filesize=11.9 K 2024-11-27T11:37:20,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/f6b49a6bacb44c6282e5bf970cf80911 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/f6b49a6bacb44c6282e5bf970cf80911 2024-11-27T11:37:20,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/f6b49a6bacb44c6282e5bf970cf80911, entries=100, sequenceid=261, filesize=9.6 K 2024-11-27T11:37:20,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/37b0aa98dc6e4d198edc1bfc78998b19 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/37b0aa98dc6e4d198edc1bfc78998b19 2024-11-27T11:37:20,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/37b0aa98dc6e4d198edc1bfc78998b19, entries=100, sequenceid=261, filesize=9.6 K 2024-11-27T11:37:20,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 6d1385241f880cf54f36aa04a79ba3c4 in 1251ms, sequenceid=261, compaction requested=true 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:20,446 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-27T11:37:20,447 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:20,447 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:20,447 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:20,447 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:20,447 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:20,447 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,447 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,447 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b917a2ada59f46acb94f8de9939b8e2c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/77ef391a2ea64f5189766b9c4a41d0d1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/f6b49a6bacb44c6282e5bf970cf80911] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=33.8 K 2024-11-27T11:37:20,447 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8d2c0776cc454b5cafa26bac6d2aee3b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b1b134cde87840be9cb00ce39fa0f96d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0d9d6b86e82c4e7e97913961bfe565eb] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.2 K 2024-11-27T11:37:20,448 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b917a2ada59f46acb94f8de9939b8e2c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732707437826 2024-11-27T11:37:20,448 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d2c0776cc454b5cafa26bac6d2aee3b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732707437826 2024-11-27T11:37:20,448 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ef391a2ea64f5189766b9c4a41d0d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732707437871 2024-11-27T11:37:20,448 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1b134cde87840be9cb00ce39fa0f96d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732707437871 2024-11-27T11:37:20,448 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9d6b86e82c4e7e97913961bfe565eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732707439084 2024-11-27T11:37:20,448 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b49a6bacb44c6282e5bf970cf80911, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732707439084 2024-11-27T11:37:20,458 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#385 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:20,458 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c263db17e40e41c4985baacb14c26eeb is 50, key is test_row_0/B:col10/1732707439084/Put/seqid=0 2024-11-27T11:37:20,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-27T11:37:20,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,461 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#386 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:20,461 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:37:20,461 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/c95bca451f5e4327bf22c122ab0fec29 is 50, key is test_row_0/A:col10/1732707439084/Put/seqid=0 2024-11-27T11:37:20,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742280_1456 (size=12849) 2024-11-27T11:37:20,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742281_1457 (size=12849) 2024-11-27T11:37:20,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/51e92e6f75dc4048aa27848fd4f4f56c is 50, key is test_row_0/A:col10/1732707439231/Put/seqid=0 2024-11-27T11:37:20,471 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/c95bca451f5e4327bf22c122ab0fec29 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c95bca451f5e4327bf22c122ab0fec29 2024-11-27T11:37:20,472 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/c263db17e40e41c4985baacb14c26eeb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c263db17e40e41c4985baacb14c26eeb 2024-11-27T11:37:20,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742282_1458 (size=12301) 2024-11-27T11:37:20,475 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/51e92e6f75dc4048aa27848fd4f4f56c 2024-11-27T11:37:20,476 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into c95bca451f5e4327bf22c122ab0fec29(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:20,477 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:20,477 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707440446; duration=0sec 2024-11-27T11:37:20,477 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:20,477 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:20,477 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:20,477 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into c263db17e40e41c4985baacb14c26eeb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:20,477 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:20,477 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707440446; duration=0sec 2024-11-27T11:37:20,478 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:20,478 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:20,479 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34655 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:20,479 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:20,479 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:20,479 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/057f877bb4274e7fa501e267713712d9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a7d223660b5940c2ba6b2657cbfd5529, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/37b0aa98dc6e4d198edc1bfc78998b19] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=33.8 K 2024-11-27T11:37:20,479 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057f877bb4274e7fa501e267713712d9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=221, earliestPutTs=1732707437826 2024-11-27T11:37:20,480 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7d223660b5940c2ba6b2657cbfd5529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1732707437871 2024-11-27T11:37:20,480 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b0aa98dc6e4d198edc1bfc78998b19, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732707439084 2024-11-27T11:37:20,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/98ac3e7240c445d3a8c0b028ec336660 is 50, key is test_row_0/B:col10/1732707439231/Put/seqid=0 2024-11-27T11:37:20,486 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:20,487 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/98c0dbf8b1474c57835945c6d42a39eb is 50, key is test_row_0/C:col10/1732707439084/Put/seqid=0 2024-11-27T11:37:20,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742284_1460 (size=12849) 2024-11-27T11:37:20,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742283_1459 (size=12301) 2024-11-27T11:37:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T11:37:20,894 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/98ac3e7240c445d3a8c0b028ec336660 2024-11-27T11:37:20,899 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/98c0dbf8b1474c57835945c6d42a39eb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/98c0dbf8b1474c57835945c6d42a39eb 2024-11-27T11:37:20,903 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 98c0dbf8b1474c57835945c6d42a39eb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:20,903 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:20,903 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707440446; duration=0sec 2024-11-27T11:37:20,903 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:20,903 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:20,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/8599a3fa0ab94e6299638096960bbde1 is 50, key is test_row_0/C:col10/1732707439231/Put/seqid=0 2024-11-27T11:37:20,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742285_1461 (size=12301) 2024-11-27T11:37:21,312 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/8599a3fa0ab94e6299638096960bbde1 2024-11-27T11:37:21,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/51e92e6f75dc4048aa27848fd4f4f56c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/51e92e6f75dc4048aa27848fd4f4f56c 2024-11-27T11:37:21,320 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/51e92e6f75dc4048aa27848fd4f4f56c, entries=150, sequenceid=285, filesize=12.0 K 2024-11-27T11:37:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/98ac3e7240c445d3a8c0b028ec336660 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/98ac3e7240c445d3a8c0b028ec336660 2024-11-27T11:37:21,324 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/98ac3e7240c445d3a8c0b028ec336660, entries=150, sequenceid=285, filesize=12.0 K 2024-11-27T11:37:21,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/8599a3fa0ab94e6299638096960bbde1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/8599a3fa0ab94e6299638096960bbde1 2024-11-27T11:37:21,328 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/8599a3fa0ab94e6299638096960bbde1, entries=150, sequenceid=285, filesize=12.0 K 2024-11-27T11:37:21,329 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=0 B/0 for 6d1385241f880cf54f36aa04a79ba3c4 in 868ms, sequenceid=285, compaction requested=false 2024-11-27T11:37:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-27T11:37:21,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-27T11:37:21,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-27T11:37:21,332 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-27T11:37:21,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.9500 sec 2024-11-27T11:37:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:21,408 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:21,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:21,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/50b83d5cdb69469084196f30f3b882c4 is 50, key is test_row_0/A:col10/1732707441407/Put/seqid=0 2024-11-27T11:37:21,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742286_1462 (size=12297) 2024-11-27T11:37:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707501451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707501452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707501460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707501461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-27T11:37:21,533 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:21,534 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-27T11:37:21,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:21,536 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:21,537 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:21,537 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:21,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707501572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707501572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707501572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707501583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:21,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707501782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707501783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707501783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:21,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707501792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:21,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:21,821 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/50b83d5cdb69469084196f30f3b882c4 2024-11-27T11:37:21,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b6cdbb5bf306459ab1b49797b68a71f6 is 50, key is test_row_0/B:col10/1732707441407/Put/seqid=0 2024-11-27T11:37:21,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742287_1463 (size=9857) 2024-11-27T11:37:21,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:21,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:21,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:21,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707502092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707502092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,100 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707502102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707502103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:22,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b6cdbb5bf306459ab1b49797b68a71f6 2024-11-27T11:37:22,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/3f460057a70a40c280c707f1695f2352 is 50, key is test_row_0/C:col10/1732707441407/Put/seqid=0 2024-11-27T11:37:22,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742288_1464 (size=9857) 2024-11-27T11:37:22,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707502602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707502602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,612 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707502612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:22,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707502623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:22,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/3f460057a70a40c280c707f1695f2352 2024-11-27T11:37:22,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/50b83d5cdb69469084196f30f3b882c4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/50b83d5cdb69469084196f30f3b882c4 2024-11-27T11:37:22,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/50b83d5cdb69469084196f30f3b882c4, entries=150, sequenceid=299, filesize=12.0 K 2024-11-27T11:37:22,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b6cdbb5bf306459ab1b49797b68a71f6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b6cdbb5bf306459ab1b49797b68a71f6 2024-11-27T11:37:22,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b6cdbb5bf306459ab1b49797b68a71f6, entries=100, sequenceid=299, filesize=9.6 K 2024-11-27T11:37:22,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/3f460057a70a40c280c707f1695f2352 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/3f460057a70a40c280c707f1695f2352 2024-11-27T11:37:22,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/3f460057a70a40c280c707f1695f2352, entries=100, sequenceid=299, filesize=9.6 K 2024-11-27T11:37:22,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6d1385241f880cf54f36aa04a79ba3c4 in 1253ms, sequenceid=299, compaction requested=true 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:22,661 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:22,661 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:22,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:22,662 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37447 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:22,662 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:22,662 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:22,662 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:22,662 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,662 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,662 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c95bca451f5e4327bf22c122ab0fec29, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/51e92e6f75dc4048aa27848fd4f4f56c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/50b83d5cdb69469084196f30f3b882c4] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.6 K 2024-11-27T11:37:22,662 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c263db17e40e41c4985baacb14c26eeb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/98ac3e7240c445d3a8c0b028ec336660, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b6cdbb5bf306459ab1b49797b68a71f6] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=34.2 K 2024-11-27T11:37:22,663 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c95bca451f5e4327bf22c122ab0fec29, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732707437871 2024-11-27T11:37:22,663 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c263db17e40e41c4985baacb14c26eeb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732707437871 2024-11-27T11:37:22,663 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51e92e6f75dc4048aa27848fd4f4f56c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732707439227 2024-11-27T11:37:22,663 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 98ac3e7240c445d3a8c0b028ec336660, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732707439227 2024-11-27T11:37:22,663 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50b83d5cdb69469084196f30f3b882c4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732707441404 2024-11-27T11:37:22,663 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b6cdbb5bf306459ab1b49797b68a71f6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732707441404 2024-11-27T11:37:22,670 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#394 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:22,670 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/52b76fd7fdfb43ab818865b0a8e57e22 is 50, key is test_row_0/B:col10/1732707441407/Put/seqid=0 2024-11-27T11:37:22,672 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:22,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:22,673 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/c9e64d83e29c4904ba8354e6e54be6f7 is 50, key is test_row_0/A:col10/1732707441407/Put/seqid=0 2024-11-27T11:37:22,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742289_1465 (size=13051) 2024-11-27T11:37:22,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742290_1466 (size=13051) 2024-11-27T11:37:22,681 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/52b76fd7fdfb43ab818865b0a8e57e22 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/52b76fd7fdfb43ab818865b0a8e57e22 2024-11-27T11:37:22,687 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into 52b76fd7fdfb43ab818865b0a8e57e22(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:22,687 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:22,687 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707442661; duration=0sec 2024-11-27T11:37:22,687 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:22,687 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:22,687 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:22,688 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35007 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:22,688 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:22,688 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,688 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/98c0dbf8b1474c57835945c6d42a39eb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/8599a3fa0ab94e6299638096960bbde1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/3f460057a70a40c280c707f1695f2352] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=34.2 K 2024-11-27T11:37:22,688 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 98c0dbf8b1474c57835945c6d42a39eb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732707437871 2024-11-27T11:37:22,689 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8599a3fa0ab94e6299638096960bbde1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1732707439227 2024-11-27T11:37:22,689 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f460057a70a40c280c707f1695f2352, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732707441404 2024-11-27T11:37:22,695 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:22,695 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/ab72bb452e79470495aa4f7afdfe1052 is 50, key is test_row_0/C:col10/1732707441407/Put/seqid=0 2024-11-27T11:37:22,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742291_1467 (size=13051) 2024-11-27T11:37:22,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-27T11:37:22,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:22,761 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:37:22,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/228f07c0bc3c44e7981b8ddf4619d8d1 is 50, key is test_row_0/A:col10/1732707441437/Put/seqid=0 2024-11-27T11:37:22,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742292_1468 (size=12301) 2024-11-27T11:37:23,083 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/c9e64d83e29c4904ba8354e6e54be6f7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c9e64d83e29c4904ba8354e6e54be6f7 2024-11-27T11:37:23,088 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into c9e64d83e29c4904ba8354e6e54be6f7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:23,088 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:23,088 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707442661; duration=0sec 2024-11-27T11:37:23,088 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:23,088 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:23,103 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/ab72bb452e79470495aa4f7afdfe1052 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ab72bb452e79470495aa4f7afdfe1052 2024-11-27T11:37:23,107 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into ab72bb452e79470495aa4f7afdfe1052(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:23,107 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:23,107 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707442661; duration=0sec 2024-11-27T11:37:23,107 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:23,107 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:23,170 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/228f07c0bc3c44e7981b8ddf4619d8d1 2024-11-27T11:37:23,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/891376465c3d4408aacb3996156378c2 is 50, key is test_row_0/B:col10/1732707441437/Put/seqid=0 2024-11-27T11:37:23,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742293_1469 (size=12301) 2024-11-27T11:37:23,582 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/891376465c3d4408aacb3996156378c2 2024-11-27T11:37:23,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1aaaa1d8df9f4d08a3554c56d1126aec is 50, key is test_row_0/C:col10/1732707441437/Put/seqid=0 2024-11-27T11:37:23,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742294_1470 (size=12301) 2024-11-27T11:37:23,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:23,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. as already flushing 2024-11-27T11:37:23,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707503636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707503638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707503640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707503641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707503752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707503752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,757 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707503752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707503762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34686 deadline: 1732707503913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,915 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 6d1385241f880cf54f36aa04a79ba3c4 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18260 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,915 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707503962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707503962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707503962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:23,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707503972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:23,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:23,994 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1aaaa1d8df9f4d08a3554c56d1126aec 2024-11-27T11:37:23,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/228f07c0bc3c44e7981b8ddf4619d8d1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/228f07c0bc3c44e7981b8ddf4619d8d1 2024-11-27T11:37:24,002 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/228f07c0bc3c44e7981b8ddf4619d8d1, entries=150, sequenceid=325, filesize=12.0 K 2024-11-27T11:37:24,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/891376465c3d4408aacb3996156378c2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/891376465c3d4408aacb3996156378c2 2024-11-27T11:37:24,005 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/891376465c3d4408aacb3996156378c2, entries=150, sequenceid=325, filesize=12.0 K 2024-11-27T11:37:24,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/1aaaa1d8df9f4d08a3554c56d1126aec as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1aaaa1d8df9f4d08a3554c56d1126aec 2024-11-27T11:37:24,009 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1aaaa1d8df9f4d08a3554c56d1126aec, entries=150, sequenceid=325, filesize=12.0 K 2024-11-27T11:37:24,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 6d1385241f880cf54f36aa04a79ba3c4 in 1249ms, sequenceid=325, compaction requested=false 2024-11-27T11:37:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:24,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-27T11:37:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-27T11:37:24,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-27T11:37:24,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4740 sec 2024-11-27T11:37:24,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.4800 sec 2024-11-27T11:37:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:24,278 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:24,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:37:24,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/45044b9508d64d6e9193f86482e69ea9 is 50, key is test_row_0/A:col10/1732707443639/Put/seqid=0 2024-11-27T11:37:24,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742295_1471 (size=12301) 2024-11-27T11:37:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707504326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707504326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707504327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707504337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,338 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707504442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707504442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707504443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,450 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707504443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707504652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707504653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,660 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707504663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707504663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/45044b9508d64d6e9193f86482e69ea9 2024-11-27T11:37:24,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b0c0adae587849e0803d6b8bf51d354e is 50, key is test_row_0/B:col10/1732707443639/Put/seqid=0 2024-11-27T11:37:24,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742296_1472 (size=12301) 2024-11-27T11:37:24,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707504962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707504962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707504983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:24,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707504983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:24,992 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:25,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b0c0adae587849e0803d6b8bf51d354e 2024-11-27T11:37:25,112 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/a700931675aa4239a36e37ab8604957c is 50, key is test_row_0/C:col10/1732707443639/Put/seqid=0 2024-11-27T11:37:25,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742297_1473 (size=12301) 2024-11-27T11:37:25,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34640 deadline: 1732707505472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:25,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34614 deadline: 1732707505482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:25,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34612 deadline: 1732707505493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:25,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34626 deadline: 1732707505503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 , the old value is region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:25,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., hostname=5b617c7edc8e,35461,1732707299005, seqNum=2 because the exception is null or not the one we care about 2024-11-27T11:37:25,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/a700931675aa4239a36e37ab8604957c 2024-11-27T11:37:25,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/45044b9508d64d6e9193f86482e69ea9 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/45044b9508d64d6e9193f86482e69ea9 2024-11-27T11:37:25,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/45044b9508d64d6e9193f86482e69ea9, entries=150, sequenceid=340, filesize=12.0 K 2024-11-27T11:37:25,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/b0c0adae587849e0803d6b8bf51d354e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b0c0adae587849e0803d6b8bf51d354e 2024-11-27T11:37:25,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b0c0adae587849e0803d6b8bf51d354e, entries=150, sequenceid=340, filesize=12.0 K 2024-11-27T11:37:25,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/a700931675aa4239a36e37ab8604957c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a700931675aa4239a36e37ab8604957c 2024-11-27T11:37:25,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a700931675aa4239a36e37ab8604957c, entries=150, sequenceid=340, filesize=12.0 K 2024-11-27T11:37:25,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6d1385241f880cf54f36aa04a79ba3c4 in 1255ms, sequenceid=340, compaction requested=true 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:25,533 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6d1385241f880cf54f36aa04a79ba3c4:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:25,533 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:25,533 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:25,534 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:25,534 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:25,534 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/A is initiating minor compaction (all files) 2024-11-27T11:37:25,534 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/B is initiating minor compaction (all files) 2024-11-27T11:37:25,534 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/A in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:25,534 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/B in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:25,534 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c9e64d83e29c4904ba8354e6e54be6f7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/228f07c0bc3c44e7981b8ddf4619d8d1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/45044b9508d64d6e9193f86482e69ea9] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.8 K 2024-11-27T11:37:25,534 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/52b76fd7fdfb43ab818865b0a8e57e22, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/891376465c3d4408aacb3996156378c2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b0c0adae587849e0803d6b8bf51d354e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.8 K 2024-11-27T11:37:25,535 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 52b76fd7fdfb43ab818865b0a8e57e22, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732707439227 2024-11-27T11:37:25,535 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9e64d83e29c4904ba8354e6e54be6f7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732707439227 2024-11-27T11:37:25,535 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 228f07c0bc3c44e7981b8ddf4619d8d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732707441437 2024-11-27T11:37:25,535 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 891376465c3d4408aacb3996156378c2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732707441437 2024-11-27T11:37:25,535 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b0c0adae587849e0803d6b8bf51d354e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707443639 2024-11-27T11:37:25,535 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45044b9508d64d6e9193f86482e69ea9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707443639 2024-11-27T11:37:25,541 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#A#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:25,541 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#B#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:25,542 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/9a5a11f29eb04ca783d5881e46ed71b5 is 50, key is test_row_0/A:col10/1732707443639/Put/seqid=0 2024-11-27T11:37:25,542 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/82408a6182fc45b98cfe8379f02c3323 is 50, key is test_row_0/B:col10/1732707443639/Put/seqid=0 2024-11-27T11:37:25,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742299_1475 (size=13153) 2024-11-27T11:37:25,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742298_1474 (size=13153) 2024-11-27T11:37:25,551 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/82408a6182fc45b98cfe8379f02c3323 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/82408a6182fc45b98cfe8379f02c3323 2024-11-27T11:37:25,555 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/B of 6d1385241f880cf54f36aa04a79ba3c4 into 82408a6182fc45b98cfe8379f02c3323(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:25,555 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:25,555 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/B, priority=13, startTime=1732707445533; duration=0sec 2024-11-27T11:37:25,556 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:25,556 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:B 2024-11-27T11:37:25,556 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:25,556 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:25,556 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 6d1385241f880cf54f36aa04a79ba3c4/C is initiating minor compaction (all files) 2024-11-27T11:37:25,556 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6d1385241f880cf54f36aa04a79ba3c4/C in TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:25,557 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ab72bb452e79470495aa4f7afdfe1052, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1aaaa1d8df9f4d08a3554c56d1126aec, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a700931675aa4239a36e37ab8604957c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp, totalSize=36.8 K 2024-11-27T11:37:25,557 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ab72bb452e79470495aa4f7afdfe1052, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1732707439227 2024-11-27T11:37:25,557 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1aaaa1d8df9f4d08a3554c56d1126aec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1732707441437 2024-11-27T11:37:25,557 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a700931675aa4239a36e37ab8604957c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1732707443639 2024-11-27T11:37:25,565 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6d1385241f880cf54f36aa04a79ba3c4#C#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:25,565 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/4047619c350248b595bcf73e06e5572c is 50, key is test_row_0/C:col10/1732707443639/Put/seqid=0 2024-11-27T11:37:25,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742300_1476 (size=13153) 2024-11-27T11:37:25,611 INFO [Thread-1823 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1823. 2024-11-27T11:37:25,612 DEBUG [Thread-1823 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:25,612 DEBUG [Thread-1823 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,612 DEBUG [Thread-1823 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,612 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:25,612 INFO [Thread-1817 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1817. 2024-11-27T11:37:25,612 DEBUG [Thread-1817 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:25,612 DEBUG [Thread-1817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,612 DEBUG [Thread-1817 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,613 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:25,618 INFO [Thread-1821 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1821. 2024-11-27T11:37:25,618 DEBUG [Thread-1821 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:25,618 DEBUG [Thread-1821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,618 INFO [Thread-1825 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1825. 2024-11-27T11:37:25,618 DEBUG [Thread-1825 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:25,618 DEBUG [Thread-1825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,618 DEBUG [Thread-1821 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:25,618 DEBUG [Thread-1825 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,618 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:25,619 INFO [Thread-1819 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1819. 2024-11-27T11:37:25,619 DEBUG [Thread-1819 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:25,619 DEBUG [Thread-1819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,619 DEBUG [Thread-1819 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:25,619 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-27T11:37:25,703 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:25,951 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/9a5a11f29eb04ca783d5881e46ed71b5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/9a5a11f29eb04ca783d5881e46ed71b5 2024-11-27T11:37:25,955 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/A of 6d1385241f880cf54f36aa04a79ba3c4 into 9a5a11f29eb04ca783d5881e46ed71b5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:25,955 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:25,955 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/A, priority=13, startTime=1732707445533; duration=0sec 2024-11-27T11:37:25,955 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:25,955 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:A 2024-11-27T11:37:25,973 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/4047619c350248b595bcf73e06e5572c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/4047619c350248b595bcf73e06e5572c 2024-11-27T11:37:25,977 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6d1385241f880cf54f36aa04a79ba3c4/C of 6d1385241f880cf54f36aa04a79ba3c4 into 4047619c350248b595bcf73e06e5572c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:25,977 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:25,977 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., storeName=6d1385241f880cf54f36aa04a79ba3c4/C, priority=13, startTime=1732707445533; duration=0sec 2024-11-27T11:37:25,977 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:25,977 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6d1385241f880cf54f36aa04a79ba3c4:C 2024-11-27T11:37:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:26,494 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:26,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:37:26,494 INFO [Thread-1814 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1814. 2024-11-27T11:37:26,494 DEBUG [Thread-1814 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:26,494 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,494 DEBUG [Thread-1814 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,494 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:26,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/331b0d6304b8440288431a477dd6a342 is 50, key is test_row_0/A:col10/1732707446493/Put/seqid=0 2024-11-27T11:37:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742301_1477 (size=12301) 2024-11-27T11:37:26,503 INFO [Thread-1806 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1806. 2024-11-27T11:37:26,503 INFO [Thread-1812 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1812. 2024-11-27T11:37:26,503 DEBUG [Thread-1806 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:26,503 DEBUG [Thread-1806 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,503 DEBUG [Thread-1812 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:26,503 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,503 DEBUG [Thread-1806 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,503 DEBUG [Thread-1812 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:26,503 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:26,533 INFO [Thread-1808 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1808. 2024-11-27T11:37:26,534 DEBUG [Thread-1808 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:26,534 DEBUG [Thread-1808 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,534 DEBUG [Thread-1808 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:26,534 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:26,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/331b0d6304b8440288431a477dd6a342 2024-11-27T11:37:26,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/37bd78b1af364e7d801d7fd0d2867ca3 is 50, key is test_row_0/B:col10/1732707446493/Put/seqid=0 2024-11-27T11:37:26,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742302_1478 (size=12301) 2024-11-27T11:37:27,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/37bd78b1af364e7d801d7fd0d2867ca3 2024-11-27T11:37:27,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/817e6bb5ac6542ad9d3fc16c17c58f92 is 50, key is test_row_0/C:col10/1732707446493/Put/seqid=0 2024-11-27T11:37:27,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742303_1479 (size=12301) 2024-11-27T11:37:27,432 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T11:37:27,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/817e6bb5ac6542ad9d3fc16c17c58f92 2024-11-27T11:37:27,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/331b0d6304b8440288431a477dd6a342 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/331b0d6304b8440288431a477dd6a342 2024-11-27T11:37:27,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/331b0d6304b8440288431a477dd6a342, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T11:37:27,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/37bd78b1af364e7d801d7fd0d2867ca3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/37bd78b1af364e7d801d7fd0d2867ca3 2024-11-27T11:37:27,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/37bd78b1af364e7d801d7fd0d2867ca3, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T11:37:27,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/817e6bb5ac6542ad9d3fc16c17c58f92 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/817e6bb5ac6542ad9d3fc16c17c58f92 2024-11-27T11:37:27,731 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/817e6bb5ac6542ad9d3fc16c17c58f92, entries=150, sequenceid=367, filesize=12.0 K 2024-11-27T11:37:27,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=20.13 KB/20610 for 6d1385241f880cf54f36aa04a79ba3c4 in 1237ms, sequenceid=367, compaction requested=false 2024-11-27T11:37:27,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6d1385241f880cf54f36aa04a79ba3c4: 2024-11-27T11:37:33,984 INFO [Thread-1810 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1810. 2024-11-27T11:37:33,984 DEBUG [Thread-1810 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:33,984 DEBUG [Thread-1810 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:33,984 DEBUG [Thread-1810 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:33,984 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:33,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T11:37:33,984 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1438 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4314 rows 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1445 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4334 rows 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1440 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4320 rows 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1428 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4282 rows 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1434 2024-11-27T11:37:33,985 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4300 rows 2024-11-27T11:37:33,985 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:37:33,985 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:37:33,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:33,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:33,985 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:33,986 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T11:37:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T11:37:33,990 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707453990"}]},"ts":"1732707453990"} 2024-11-27T11:37:33,991 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T11:37:33,991 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T11:37:33,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:37:33,993 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, UNASSIGN}] 2024-11-27T11:37:33,993 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, UNASSIGN 2024-11-27T11:37:33,994 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=6d1385241f880cf54f36aa04a79ba3c4, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:33,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, UNASSIGN because future has completed 2024-11-27T11:37:33,995 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:37:33,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:37:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T11:37:34,147 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(122): Close 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:34,147 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:37:34,147 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1722): Closing 6d1385241f880cf54f36aa04a79ba3c4, disabling compactions & flushes 2024-11-27T11:37:34,147 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:34,147 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:34,147 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. after waiting 0 ms 2024-11-27T11:37:34,147 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:34,148 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(2902): Flushing 6d1385241f880cf54f36aa04a79ba3c4 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-27T11:37:34,151 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/32f8c4297c974f3a9af3f5e77087171b is 50, key is test_row_0/A:col10/1732707446502/Put/seqid=0 2024-11-27T11:37:34,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742304_1480 (size=12301) 2024-11-27T11:37:34,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T11:37:34,554 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/32f8c4297c974f3a9af3f5e77087171b 2024-11-27T11:37:34,560 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a9b568e7d9c245a5bae3d615f3df2371 is 50, key is test_row_0/B:col10/1732707446502/Put/seqid=0 2024-11-27T11:37:34,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742305_1481 (size=12301) 2024-11-27T11:37:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T11:37:34,963 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a9b568e7d9c245a5bae3d615f3df2371 2024-11-27T11:37:34,969 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/f7a6bab832114aad90708aac4c263d23 is 50, key is test_row_0/C:col10/1732707446502/Put/seqid=0 2024-11-27T11:37:34,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742306_1482 (size=12301) 2024-11-27T11:37:35,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T11:37:35,372 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/f7a6bab832114aad90708aac4c263d23 2024-11-27T11:37:35,375 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/A/32f8c4297c974f3a9af3f5e77087171b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/32f8c4297c974f3a9af3f5e77087171b 2024-11-27T11:37:35,378 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/32f8c4297c974f3a9af3f5e77087171b, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T11:37:35,379 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/B/a9b568e7d9c245a5bae3d615f3df2371 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9b568e7d9c245a5bae3d615f3df2371 2024-11-27T11:37:35,381 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9b568e7d9c245a5bae3d615f3df2371, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T11:37:35,382 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/.tmp/C/f7a6bab832114aad90708aac4c263d23 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f7a6bab832114aad90708aac4c263d23 2024-11-27T11:37:35,384 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f7a6bab832114aad90708aac4c263d23, entries=150, sequenceid=374, filesize=12.0 K 2024-11-27T11:37:35,385 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6d1385241f880cf54f36aa04a79ba3c4 in 1238ms, sequenceid=374, compaction requested=true 2024-11-27T11:37:35,385 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/30f23e353a8d4e9fb2b56a9b81d5ce90, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/fecdfeff876b41bc95e0c24d4e48d474, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8c280d90ee45428e9559050d0e2c1558, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/02f36d7572d74f39b3fab1a7c8682e5f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0aa5831eac804e88a85ae5c3f2e69547, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/891ffd1b66b7479fb36e7a037d871e18, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/aee2a513e4ac4edb92254c9aab5e15fb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/06564325a7cb40dd90cefdeaa6d069da, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/40d8b173644042aa9c32c4db8cf4a4af, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b0fad81de1e8404c9d12bdceec06efee, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/212a8afe8b834734a7e80045005e2a68, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/46f1af0ad0c344b4bc9adf8464543deb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/4c36aaf8e2394cfb9069513525d7a873, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/64db0b231fcb440995978264922349ad, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/370a5da61bad44e8a6e60e8cc468b2a1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/404fc64d597b4d0bae61fe0646643cf6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8d2c0776cc454b5cafa26bac6d2aee3b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b1b134cde87840be9cb00ce39fa0f96d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c95bca451f5e4327bf22c122ab0fec29, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0d9d6b86e82c4e7e97913961bfe565eb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/51e92e6f75dc4048aa27848fd4f4f56c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c9e64d83e29c4904ba8354e6e54be6f7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/50b83d5cdb69469084196f30f3b882c4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/228f07c0bc3c44e7981b8ddf4619d8d1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/45044b9508d64d6e9193f86482e69ea9] to archive 2024-11-27T11:37:35,386 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:37:35,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/30f23e353a8d4e9fb2b56a9b81d5ce90 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/30f23e353a8d4e9fb2b56a9b81d5ce90 2024-11-27T11:37:35,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/fecdfeff876b41bc95e0c24d4e48d474 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/fecdfeff876b41bc95e0c24d4e48d474 2024-11-27T11:37:35,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8c280d90ee45428e9559050d0e2c1558 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8c280d90ee45428e9559050d0e2c1558 2024-11-27T11:37:35,389 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/02f36d7572d74f39b3fab1a7c8682e5f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/02f36d7572d74f39b3fab1a7c8682e5f 2024-11-27T11:37:35,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0aa5831eac804e88a85ae5c3f2e69547 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0aa5831eac804e88a85ae5c3f2e69547 2024-11-27T11:37:35,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/891ffd1b66b7479fb36e7a037d871e18 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/891ffd1b66b7479fb36e7a037d871e18 2024-11-27T11:37:35,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/aee2a513e4ac4edb92254c9aab5e15fb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/aee2a513e4ac4edb92254c9aab5e15fb 2024-11-27T11:37:35,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/06564325a7cb40dd90cefdeaa6d069da to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/06564325a7cb40dd90cefdeaa6d069da 2024-11-27T11:37:35,392 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/40d8b173644042aa9c32c4db8cf4a4af to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/40d8b173644042aa9c32c4db8cf4a4af 2024-11-27T11:37:35,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b0fad81de1e8404c9d12bdceec06efee to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b0fad81de1e8404c9d12bdceec06efee 2024-11-27T11:37:35,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/212a8afe8b834734a7e80045005e2a68 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/212a8afe8b834734a7e80045005e2a68 2024-11-27T11:37:35,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/46f1af0ad0c344b4bc9adf8464543deb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/46f1af0ad0c344b4bc9adf8464543deb 2024-11-27T11:37:35,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/4c36aaf8e2394cfb9069513525d7a873 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/4c36aaf8e2394cfb9069513525d7a873 2024-11-27T11:37:35,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/64db0b231fcb440995978264922349ad to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/64db0b231fcb440995978264922349ad 2024-11-27T11:37:35,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/370a5da61bad44e8a6e60e8cc468b2a1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/370a5da61bad44e8a6e60e8cc468b2a1 2024-11-27T11:37:35,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/404fc64d597b4d0bae61fe0646643cf6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/404fc64d597b4d0bae61fe0646643cf6 2024-11-27T11:37:35,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8d2c0776cc454b5cafa26bac6d2aee3b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/8d2c0776cc454b5cafa26bac6d2aee3b 2024-11-27T11:37:35,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b1b134cde87840be9cb00ce39fa0f96d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/b1b134cde87840be9cb00ce39fa0f96d 2024-11-27T11:37:35,400 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c95bca451f5e4327bf22c122ab0fec29 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c95bca451f5e4327bf22c122ab0fec29 2024-11-27T11:37:35,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0d9d6b86e82c4e7e97913961bfe565eb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/0d9d6b86e82c4e7e97913961bfe565eb 2024-11-27T11:37:35,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/51e92e6f75dc4048aa27848fd4f4f56c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/51e92e6f75dc4048aa27848fd4f4f56c 2024-11-27T11:37:35,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c9e64d83e29c4904ba8354e6e54be6f7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/c9e64d83e29c4904ba8354e6e54be6f7 2024-11-27T11:37:35,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/50b83d5cdb69469084196f30f3b882c4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/50b83d5cdb69469084196f30f3b882c4 2024-11-27T11:37:35,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/228f07c0bc3c44e7981b8ddf4619d8d1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/228f07c0bc3c44e7981b8ddf4619d8d1 2024-11-27T11:37:35,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/45044b9508d64d6e9193f86482e69ea9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/45044b9508d64d6e9193f86482e69ea9 2024-11-27T11:37:35,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c2070b5ebc294d1595331cbcadaa9961, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/ab6ce3bc8ca14cd49e37677382a07a2c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a86990deb348487083919fba6898583e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/6d4448294fe34e32b488dfd76aef6a4a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/84aceb4b93c644d3bb636ecf4e749049, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9ccd3522886400aa918ed443fef8121, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c709dcff7fc54b529e1aecb729e717e0, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/3b759618a17f4795b4414e4828ac146e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/1d3467e40b2244b1ab405a2186f2dd44, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a2f8a76f4dd94fc490c50b06ab174f2e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/88ce04880fba4fb283be675fdda0b780, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/818ee73c185440a6b3516ef32551003c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/5bb483a8714c422e9c4c946dbcdd93a7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/4b7c3965fadf490db1cdf0a502e1702d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/dcc3994ac1df4bffbc4d7afa85cdf8c9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b917a2ada59f46acb94f8de9939b8e2c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b5b7aaf8a61545de9cf2744db83eab49, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/77ef391a2ea64f5189766b9c4a41d0d1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c263db17e40e41c4985baacb14c26eeb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/f6b49a6bacb44c6282e5bf970cf80911, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/98ac3e7240c445d3a8c0b028ec336660, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/52b76fd7fdfb43ab818865b0a8e57e22, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b6cdbb5bf306459ab1b49797b68a71f6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/891376465c3d4408aacb3996156378c2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b0c0adae587849e0803d6b8bf51d354e] to archive 2024-11-27T11:37:35,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:37:35,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c2070b5ebc294d1595331cbcadaa9961 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c2070b5ebc294d1595331cbcadaa9961 2024-11-27T11:37:35,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/ab6ce3bc8ca14cd49e37677382a07a2c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/ab6ce3bc8ca14cd49e37677382a07a2c 2024-11-27T11:37:35,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a86990deb348487083919fba6898583e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a86990deb348487083919fba6898583e 2024-11-27T11:37:35,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/6d4448294fe34e32b488dfd76aef6a4a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/6d4448294fe34e32b488dfd76aef6a4a 2024-11-27T11:37:35,410 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/84aceb4b93c644d3bb636ecf4e749049 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/84aceb4b93c644d3bb636ecf4e749049 2024-11-27T11:37:35,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9ccd3522886400aa918ed443fef8121 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9ccd3522886400aa918ed443fef8121 2024-11-27T11:37:35,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c709dcff7fc54b529e1aecb729e717e0 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c709dcff7fc54b529e1aecb729e717e0 2024-11-27T11:37:35,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/3b759618a17f4795b4414e4828ac146e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/3b759618a17f4795b4414e4828ac146e 2024-11-27T11:37:35,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/1d3467e40b2244b1ab405a2186f2dd44 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/1d3467e40b2244b1ab405a2186f2dd44 2024-11-27T11:37:35,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a2f8a76f4dd94fc490c50b06ab174f2e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a2f8a76f4dd94fc490c50b06ab174f2e 2024-11-27T11:37:35,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/88ce04880fba4fb283be675fdda0b780 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/88ce04880fba4fb283be675fdda0b780 2024-11-27T11:37:35,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/818ee73c185440a6b3516ef32551003c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/818ee73c185440a6b3516ef32551003c 2024-11-27T11:37:35,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/5bb483a8714c422e9c4c946dbcdd93a7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/5bb483a8714c422e9c4c946dbcdd93a7 2024-11-27T11:37:35,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/4b7c3965fadf490db1cdf0a502e1702d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/4b7c3965fadf490db1cdf0a502e1702d 2024-11-27T11:37:35,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/dcc3994ac1df4bffbc4d7afa85cdf8c9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/dcc3994ac1df4bffbc4d7afa85cdf8c9 2024-11-27T11:37:35,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b917a2ada59f46acb94f8de9939b8e2c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b917a2ada59f46acb94f8de9939b8e2c 2024-11-27T11:37:35,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b5b7aaf8a61545de9cf2744db83eab49 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b5b7aaf8a61545de9cf2744db83eab49 2024-11-27T11:37:35,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/77ef391a2ea64f5189766b9c4a41d0d1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/77ef391a2ea64f5189766b9c4a41d0d1 2024-11-27T11:37:35,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c263db17e40e41c4985baacb14c26eeb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/c263db17e40e41c4985baacb14c26eeb 2024-11-27T11:37:35,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/f6b49a6bacb44c6282e5bf970cf80911 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/f6b49a6bacb44c6282e5bf970cf80911 2024-11-27T11:37:35,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/98ac3e7240c445d3a8c0b028ec336660 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/98ac3e7240c445d3a8c0b028ec336660 2024-11-27T11:37:35,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/52b76fd7fdfb43ab818865b0a8e57e22 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/52b76fd7fdfb43ab818865b0a8e57e22 2024-11-27T11:37:35,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b6cdbb5bf306459ab1b49797b68a71f6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b6cdbb5bf306459ab1b49797b68a71f6 2024-11-27T11:37:35,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/891376465c3d4408aacb3996156378c2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/891376465c3d4408aacb3996156378c2 2024-11-27T11:37:35,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b0c0adae587849e0803d6b8bf51d354e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/b0c0adae587849e0803d6b8bf51d354e 2024-11-27T11:37:35,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/d879816eae56457787bf926db4a03903, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/14134ebe15ef43db8f09a24cf4b17bef, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1d642511306a446e94d9eb5555fd5dbf, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/71313017addd4980a372ab751d6b7e5b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/320de7b487f54b4980401030dffc2f71, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/642b2a3d53d14b7986a955b56bf627d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/5bd12fa4fa734f5a8efc1a298c2f7bf7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/efb2760f92894c4c84e214ffa66735ff, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/47886ce158ef40d19942ac13f6b3c077, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1679b05c5df24847b724d271d3091924, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f8de38d28c4b4f3fa39d441da48c1ee4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/164ce1259b4d47a0afde43143f122517, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/2655126152174d23b5b21e2c8dd5b993, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/79d197d435724a4f9d80095fc36d3f7e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/97b92ddd025042c3afc3d7f1ee0f70bf, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/057f877bb4274e7fa501e267713712d9, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ead93a979a234877b29703279c2b439f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a7d223660b5940c2ba6b2657cbfd5529, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/98c0dbf8b1474c57835945c6d42a39eb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/37b0aa98dc6e4d198edc1bfc78998b19, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/8599a3fa0ab94e6299638096960bbde1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ab72bb452e79470495aa4f7afdfe1052, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/3f460057a70a40c280c707f1695f2352, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1aaaa1d8df9f4d08a3554c56d1126aec, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a700931675aa4239a36e37ab8604957c] to archive 2024-11-27T11:37:35,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:37:35,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/d879816eae56457787bf926db4a03903 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/d879816eae56457787bf926db4a03903 2024-11-27T11:37:35,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/14134ebe15ef43db8f09a24cf4b17bef to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/14134ebe15ef43db8f09a24cf4b17bef 2024-11-27T11:37:35,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1d642511306a446e94d9eb5555fd5dbf to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1d642511306a446e94d9eb5555fd5dbf 2024-11-27T11:37:35,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/71313017addd4980a372ab751d6b7e5b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/71313017addd4980a372ab751d6b7e5b 2024-11-27T11:37:35,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/320de7b487f54b4980401030dffc2f71 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/320de7b487f54b4980401030dffc2f71 2024-11-27T11:37:35,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/642b2a3d53d14b7986a955b56bf627d5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/642b2a3d53d14b7986a955b56bf627d5 2024-11-27T11:37:35,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/5bd12fa4fa734f5a8efc1a298c2f7bf7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/5bd12fa4fa734f5a8efc1a298c2f7bf7 2024-11-27T11:37:35,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/efb2760f92894c4c84e214ffa66735ff to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/efb2760f92894c4c84e214ffa66735ff 2024-11-27T11:37:35,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/47886ce158ef40d19942ac13f6b3c077 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/47886ce158ef40d19942ac13f6b3c077 2024-11-27T11:37:35,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1679b05c5df24847b724d271d3091924 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1679b05c5df24847b724d271d3091924 2024-11-27T11:37:35,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f8de38d28c4b4f3fa39d441da48c1ee4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f8de38d28c4b4f3fa39d441da48c1ee4 2024-11-27T11:37:35,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/164ce1259b4d47a0afde43143f122517 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/164ce1259b4d47a0afde43143f122517 2024-11-27T11:37:35,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/2655126152174d23b5b21e2c8dd5b993 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/2655126152174d23b5b21e2c8dd5b993 2024-11-27T11:37:35,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/79d197d435724a4f9d80095fc36d3f7e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/79d197d435724a4f9d80095fc36d3f7e 2024-11-27T11:37:35,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/97b92ddd025042c3afc3d7f1ee0f70bf to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/97b92ddd025042c3afc3d7f1ee0f70bf 2024-11-27T11:37:35,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/057f877bb4274e7fa501e267713712d9 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/057f877bb4274e7fa501e267713712d9 2024-11-27T11:37:35,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ead93a979a234877b29703279c2b439f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ead93a979a234877b29703279c2b439f 2024-11-27T11:37:35,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a7d223660b5940c2ba6b2657cbfd5529 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a7d223660b5940c2ba6b2657cbfd5529 2024-11-27T11:37:35,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/98c0dbf8b1474c57835945c6d42a39eb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/98c0dbf8b1474c57835945c6d42a39eb 2024-11-27T11:37:35,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/37b0aa98dc6e4d198edc1bfc78998b19 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/37b0aa98dc6e4d198edc1bfc78998b19 2024-11-27T11:37:35,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/8599a3fa0ab94e6299638096960bbde1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/8599a3fa0ab94e6299638096960bbde1 2024-11-27T11:37:35,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ab72bb452e79470495aa4f7afdfe1052 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/ab72bb452e79470495aa4f7afdfe1052 2024-11-27T11:37:35,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/3f460057a70a40c280c707f1695f2352 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/3f460057a70a40c280c707f1695f2352 2024-11-27T11:37:35,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1aaaa1d8df9f4d08a3554c56d1126aec to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/1aaaa1d8df9f4d08a3554c56d1126aec 2024-11-27T11:37:35,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a700931675aa4239a36e37ab8604957c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/a700931675aa4239a36e37ab8604957c 2024-11-27T11:37:35,449 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/recovered.edits/377.seqid, newMaxSeqId=377, maxSeqId=1 2024-11-27T11:37:35,449 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. 2024-11-27T11:37:35,450 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] regionserver.HRegion(1676): Region close journal for 6d1385241f880cf54f36aa04a79ba3c4: Waiting for close lock at 1732707454147Running coprocessor pre-close hooks at 1732707454147Disabling compacts and flushes for region at 1732707454147Disabling writes for close at 1732707454147Obtaining lock to block concurrent updates at 1732707454148 (+1 ms)Preparing flush snapshotting stores in 6d1385241f880cf54f36aa04a79ba3c4 at 1732707454148Finished memstore snapshotting TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732707454148Flushing stores of TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4. at 1732707454148Flushing 6d1385241f880cf54f36aa04a79ba3c4/A: creating writer at 1732707454148Flushing 6d1385241f880cf54f36aa04a79ba3c4/A: appending metadata at 1732707454150 (+2 ms)Flushing 6d1385241f880cf54f36aa04a79ba3c4/A: closing flushed file at 1732707454151 (+1 ms)Flushing 6d1385241f880cf54f36aa04a79ba3c4/B: creating writer at 1732707454558 (+407 ms)Flushing 6d1385241f880cf54f36aa04a79ba3c4/B: appending metadata at 1732707454560 (+2 ms)Flushing 6d1385241f880cf54f36aa04a79ba3c4/B: closing flushed file at 1732707454560Flushing 6d1385241f880cf54f36aa04a79ba3c4/C: creating writer at 1732707454966 (+406 ms)Flushing 6d1385241f880cf54f36aa04a79ba3c4/C: appending metadata at 1732707454968 (+2 ms)Flushing 6d1385241f880cf54f36aa04a79ba3c4/C: closing flushed file at 1732707454968Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56988d82: reopening flushed file at 1732707455375 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60e6704b: reopening flushed file at 1732707455378 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@740e42d1: reopening flushed file at 1732707455381 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6d1385241f880cf54f36aa04a79ba3c4 in 1238ms, sequenceid=374, compaction requested=true at 1732707455385 (+4 ms)Writing region close event to WAL at 1732707455446 (+61 ms)Running coprocessor post-close hooks at 1732707455449 (+3 ms)Closed at 1732707455449 2024-11-27T11:37:35,451 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=130}] handler.UnassignRegionHandler(157): Closed 6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:35,452 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=6d1385241f880cf54f36aa04a79ba3c4, regionState=CLOSED 2024-11-27T11:37:35,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:37:35,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-27T11:37:35,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; CloseRegionProcedure 6d1385241f880cf54f36aa04a79ba3c4, server=5b617c7edc8e,35461,1732707299005 in 1.4580 sec 2024-11-27T11:37:35,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-27T11:37:35,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6d1385241f880cf54f36aa04a79ba3c4, UNASSIGN in 1.4620 sec 2024-11-27T11:37:35,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-27T11:37:35,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4640 sec 2024-11-27T11:37:35,458 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707455458"}]},"ts":"1732707455458"} 2024-11-27T11:37:35,459 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T11:37:35,459 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T11:37:35,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4740 sec 2024-11-27T11:37:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-27T11:37:36,132 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:36,133 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T11:37:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,134 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=131, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T11:37:36,135 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=131, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,136 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:36,138 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/recovered.edits] 2024-11-27T11:37:36,140 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/32f8c4297c974f3a9af3f5e77087171b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/32f8c4297c974f3a9af3f5e77087171b 2024-11-27T11:37:36,141 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/331b0d6304b8440288431a477dd6a342 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/331b0d6304b8440288431a477dd6a342 2024-11-27T11:37:36,142 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/9a5a11f29eb04ca783d5881e46ed71b5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/A/9a5a11f29eb04ca783d5881e46ed71b5 2024-11-27T11:37:36,143 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/37bd78b1af364e7d801d7fd0d2867ca3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/37bd78b1af364e7d801d7fd0d2867ca3 2024-11-27T11:37:36,144 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/82408a6182fc45b98cfe8379f02c3323 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/82408a6182fc45b98cfe8379f02c3323 2024-11-27T11:37:36,145 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9b568e7d9c245a5bae3d615f3df2371 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/B/a9b568e7d9c245a5bae3d615f3df2371 2024-11-27T11:37:36,147 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/4047619c350248b595bcf73e06e5572c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/4047619c350248b595bcf73e06e5572c 2024-11-27T11:37:36,147 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/817e6bb5ac6542ad9d3fc16c17c58f92 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/817e6bb5ac6542ad9d3fc16c17c58f92 2024-11-27T11:37:36,148 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f7a6bab832114aad90708aac4c263d23 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/C/f7a6bab832114aad90708aac4c263d23 2024-11-27T11:37:36,150 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/recovered.edits/377.seqid to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4/recovered.edits/377.seqid 2024-11-27T11:37:36,150 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/6d1385241f880cf54f36aa04a79ba3c4 2024-11-27T11:37:36,150 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T11:37:36,152 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=131, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,153 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T11:37:36,155 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T11:37:36,155 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=131, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,155 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T11:37:36,156 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732707456155"}]},"ts":"9223372036854775807"} 2024-11-27T11:37:36,157 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T11:37:36,157 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6d1385241f880cf54f36aa04a79ba3c4, NAME => 'TestAcidGuarantees,,1732707423410.6d1385241f880cf54f36aa04a79ba3c4.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T11:37:36,157 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T11:37:36,157 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732707456157"}]},"ts":"9223372036854775807"} 2024-11-27T11:37:36,158 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T11:37:36,159 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=131, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 26 msec 2024-11-27T11:37:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-27T11:37:36,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T11:37:36,243 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:36,251 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 238), OpenFileDescriptor=443 (was 447), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 278), ProcessCount=11 (was 11), AvailableMemoryMB=6336 (was 6351) 2024-11-27T11:37:36,259 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=227, ProcessCount=11, AvailableMemoryMB=6336 2024-11-27T11:37:36,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:37:36,261 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:37:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:36,262 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-27T11:37:36,262 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:36,262 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 132 2024-11-27T11:37:36,263 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-27T11:37:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-27T11:37:36,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742307_1483 (size=959) 2024-11-27T11:37:36,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-27T11:37:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-27T11:37:36,669 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b 2024-11-27T11:37:36,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742308_1484 (size=53) 2024-11-27T11:37:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-27T11:37:37,074 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:37:37,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 00f69fd5a9af317bd8000447a4cf50b8, disabling compactions & flushes 2024-11-27T11:37:37,075 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. after waiting 0 ms 2024-11-27T11:37:37,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,075 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,075 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 00f69fd5a9af317bd8000447a4cf50b8: Waiting for close lock at 1732707457075Disabling compacts and flushes for region at 1732707457075Disabling writes for close at 1732707457075Writing region close event to WAL at 1732707457075Closed at 1732707457075 2024-11-27T11:37:37,076 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-27T11:37:37,076 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732707457076"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732707457076"}]},"ts":"1732707457076"} 2024-11-27T11:37:37,077 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-27T11:37:37,078 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-27T11:37:37,078 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707457078"}]},"ts":"1732707457078"} 2024-11-27T11:37:37,079 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-27T11:37:37,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, ASSIGN}] 2024-11-27T11:37:37,080 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, ASSIGN 2024-11-27T11:37:37,080 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, ASSIGN; state=OFFLINE, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=false 2024-11-27T11:37:37,231 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:37,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, ASSIGN because future has completed 2024-11-27T11:37:37,233 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:37:37,387 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,387 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7752): Opening region: {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:37:37,387 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,387 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:37:37,387 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7794): checking encryption for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,387 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(7797): checking classloading for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,388 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,389 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f69fd5a9af317bd8000447a4cf50b8 columnFamilyName A 2024-11-27T11:37:37,389 DEBUG [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:37,390 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(327): Store=00f69fd5a9af317bd8000447a4cf50b8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:37,390 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,391 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f69fd5a9af317bd8000447a4cf50b8 columnFamilyName B 2024-11-27T11:37:37,391 DEBUG [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:37,391 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(327): Store=00f69fd5a9af317bd8000447a4cf50b8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:37,391 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,392 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f69fd5a9af317bd8000447a4cf50b8 columnFamilyName C 2024-11-27T11:37:37,392 DEBUG [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:37,392 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(327): Store=00f69fd5a9af317bd8000447a4cf50b8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:37,392 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1038): replaying wal for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,393 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,393 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,394 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1048): stopping wal replay for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,394 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1060): Cleaning up temporary data for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,394 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:37:37,395 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1093): writing seq id for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,396 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-27T11:37:37,397 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1114): Opened 00f69fd5a9af317bd8000447a4cf50b8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63068215, jitterRate=-0.06021036207675934}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:37:37,397 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:37,397 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegion(1006): Region open journal for 00f69fd5a9af317bd8000447a4cf50b8: Running coprocessor pre-open hook at 1732707457387Writing region info on filesystem at 1732707457387Initializing all the Stores at 1732707457388 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707457388Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707457388Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707457388Cleaning up temporary data from old regions at 1732707457394 (+6 ms)Running coprocessor post-open hooks at 1732707457397 (+3 ms)Region opened successfully at 1732707457397 2024-11-27T11:37:37,398 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., pid=134, masterSystemTime=1732707457384 2024-11-27T11:37:37,399 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,399 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=134}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:37,400 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=OPEN, openSeqNum=2, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:37,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:37:37,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-27T11:37:37,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 in 169 msec 2024-11-27T11:37:37,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-27T11:37:37,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-27T11:37:37,404 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, ASSIGN in 324 msec 2024-11-27T11:37:37,405 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-27T11:37:37,405 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707457405"}]},"ts":"1732707457405"} 2024-11-27T11:37:37,406 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-27T11:37:37,406 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=132, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-27T11:37:37,407 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1450 sec 2024-11-27T11:37:38,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=132 2024-11-27T11:37:38,423 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:38,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d3c5fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:38,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:38,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:38,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b17633d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:38,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50532, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f9c394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:38,427 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:38,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:38,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:38,428 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:38,429 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-27T11:37:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 5b617c7edc8e,35539,1732707298302 2024-11-27T11:37:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@134e16fc 2024-11-27T11:37:38,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-27T11:37:38,430 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50544, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-27T11:37:38,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-27T11:37:38,432 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-27T11:37:38,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-27T11:37:38,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742309_1485 (size=995) 2024-11-27T11:37:38,842 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-27T11:37:38,842 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-27T11:37:38,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:37:38,846 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, REOPEN/MOVE}] 2024-11-27T11:37:38,846 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, REOPEN/MOVE 2024-11-27T11:37:38,847 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:38,848 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, REOPEN/MOVE because future has completed 2024-11-27T11:37:38,848 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:37:38,848 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:37:39,000 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(122): Close 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,000 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:37:39,000 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1722): Closing 00f69fd5a9af317bd8000447a4cf50b8, disabling compactions & flushes 2024-11-27T11:37:39,000 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,000 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,000 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. after waiting 0 ms 2024-11-27T11:37:39,000 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,003 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-27T11:37:39,004 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,004 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegion(1676): Region close journal for 00f69fd5a9af317bd8000447a4cf50b8: Waiting for close lock at 1732707459000Running coprocessor pre-close hooks at 1732707459000Disabling compacts and flushes for region at 1732707459000Disabling writes for close at 1732707459000Writing region close event to WAL at 1732707459001 (+1 ms)Running coprocessor post-close hooks at 1732707459004 (+3 ms)Closed at 1732707459004 2024-11-27T11:37:39,004 WARN [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] regionserver.HRegionServer(3299): Not adding moved region record: 00f69fd5a9af317bd8000447a4cf50b8 to self. 2024-11-27T11:37:39,005 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=138}] handler.UnassignRegionHandler(157): Closed 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,006 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=CLOSED 2024-11-27T11:37:39,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:37:39,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-27T11:37:39,008 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 in 159 msec 2024-11-27T11:37:39,009 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, REOPEN/MOVE; state=CLOSED, location=5b617c7edc8e,35461,1732707299005; forceNewPlan=false, retain=true 2024-11-27T11:37:39,159 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=OPENING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, REOPEN/MOVE because future has completed 2024-11-27T11:37:39,161 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:37:39,315 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,315 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7752): Opening region: {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} 2024-11-27T11:37:39,315 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,315 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-27T11:37:39,315 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7794): checking encryption for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,315 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(7797): checking classloading for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,316 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,317 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f69fd5a9af317bd8000447a4cf50b8 columnFamilyName A 2024-11-27T11:37:39,318 DEBUG [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:39,318 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(327): Store=00f69fd5a9af317bd8000447a4cf50b8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:39,319 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,319 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f69fd5a9af317bd8000447a4cf50b8 columnFamilyName B 2024-11-27T11:37:39,320 DEBUG [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:39,320 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(327): Store=00f69fd5a9af317bd8000447a4cf50b8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:39,320 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,320 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 00f69fd5a9af317bd8000447a4cf50b8 columnFamilyName C 2024-11-27T11:37:39,320 DEBUG [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:39,321 INFO [StoreOpener-00f69fd5a9af317bd8000447a4cf50b8-1 {}] regionserver.HStore(327): Store=00f69fd5a9af317bd8000447a4cf50b8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-27T11:37:39,321 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1038): replaying wal for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,321 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,322 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,323 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1048): stopping wal replay for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,323 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1060): Cleaning up temporary data for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,323 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-27T11:37:39,324 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1093): writing seq id for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,324 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1114): Opened 00f69fd5a9af317bd8000447a4cf50b8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61985040, jitterRate=-0.07635092735290527}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-27T11:37:39,324 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,325 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegion(1006): Region open journal for 00f69fd5a9af317bd8000447a4cf50b8: Running coprocessor pre-open hook at 1732707459315Writing region info on filesystem at 1732707459315Initializing all the Stores at 1732707459316 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707459316Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707459316Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732707459316Cleaning up temporary data from old regions at 1732707459323 (+7 ms)Running coprocessor post-open hooks at 1732707459324 (+1 ms)Region opened successfully at 1732707459325 (+1 ms) 2024-11-27T11:37:39,326 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., pid=139, masterSystemTime=1732707459312 2024-11-27T11:37:39,327 DEBUG [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,327 INFO [RS_OPEN_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_OPEN_REGION, pid=139}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,327 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=OPEN, openSeqNum=5, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:37:39,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=137 2024-11-27T11:37:39,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 in 168 msec 2024-11-27T11:37:39,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-27T11:37:39,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, REOPEN/MOVE in 485 msec 2024-11-27T11:37:39,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-27T11:37:39,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 488 msec 2024-11-27T11:37:39,335 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-27T11:37:39,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-27T11:37:39,336 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:39,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@699866f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,338 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529853a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,339 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50562, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7babf9ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,340 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a5ce51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,340 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,340 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,340 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7caa9526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,342 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50590, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,342 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@685276cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,342 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aade4f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,342 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,343 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,343 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633b1a1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,344 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50604, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@350fc47b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,345 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48a0f4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c67d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,346 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aa7b476, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,347 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18996b84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,347 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,347 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,348 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@214204f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,348 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,349 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,349 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@676331a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57b2366d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,350 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228e08c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,352 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@888c294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@727f1adc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,353 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f6c0eba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,354 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50666, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,354 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@526486f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,355 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b554b33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,355 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,355 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,355 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76103e59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,357 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,357 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7792d5e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,357 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113f4b7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,357 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,357 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,358 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ccac39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,358 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,359 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6d7008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76341e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,360 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 5b617c7edc8e,35539,-1 for getting cluster id 2024-11-27T11:37:39,360 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-27T11:37:39,361 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '298a1ba5-84a0-4a11-841c-c5777d1b0342' 2024-11-27T11:37:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-27T11:37:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "298a1ba5-84a0-4a11-841c-c5777d1b0342" 2024-11-27T11:37:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1525ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [5b617c7edc8e,35539,-1] 2024-11-27T11:37:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-27T11:37:39,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:39,362 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-27T11:37:39,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15fde651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-27T11:37:39,363 DEBUG [Thread-2200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,363 DEBUG [Thread-2202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,363 DEBUG [Thread-2206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,363 DEBUG [Thread-2193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,363 DEBUG [Thread-2189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,363 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,364 DEBUG [Thread-2187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,364 DEBUG [Thread-2191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,365 DEBUG [Thread-2198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,365 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54608, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,366 DEBUG [Thread-2195 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54592, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,366 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54632, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:39,367 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54652, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,368 DEBUG [Thread-2204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-27T11:37:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=5b617c7edc8e,35461,1732707299005, seqNum=-1] 2024-11-27T11:37:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-27T11:37:39,369 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54662, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,369 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,371 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-27T11:37:39,372 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,372 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-27T11:37:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-27T11:37:39,372 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:39,373 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:39,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:39,374 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5] 2024-11-27T11:37:39,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:39,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:39,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bd3fcb8d4fb844f1be4183680e449b82_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707459374/Put/seqid=0 2024-11-27T11:37:39,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707519403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707519404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707519405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,409 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707519405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707519406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742310_1486 (size=12154) 2024-11-27T11:37:39,439 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:39,444 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bd3fcb8d4fb844f1be4183680e449b82_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bd3fcb8d4fb844f1be4183680e449b82_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,445 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8cf9976355844aab9ca456b4b58f0ca2, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:39,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8cf9976355844aab9ca456b4b58f0ca2 is 175, key is test_row_0/A:col10/1732707459374/Put/seqid=0 2024-11-27T11:37:39,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742311_1487 (size=30955) 2024-11-27T11:37:39,451 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8cf9976355844aab9ca456b4b58f0ca2 2024-11-27T11:37:39,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/c80027be68624da0a20732766c7d7e3f is 50, key is test_row_0/B:col10/1732707459374/Put/seqid=0 2024-11-27T11:37:39,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742312_1488 (size=12001) 2024-11-27T11:37:39,482 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/c80027be68624da0a20732766c7d7e3f 2024-11-27T11:37:39,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-27T11:37:39,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/6bf4c6e929ce4429b43a291388aae5a7 is 50, key is test_row_0/C:col10/1732707459374/Put/seqid=0 2024-11-27T11:37:39,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742313_1489 (size=12001) 2024-11-27T11:37:39,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/6bf4c6e929ce4429b43a291388aae5a7 2024-11-27T11:37:39,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8cf9976355844aab9ca456b4b58f0ca2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2 2024-11-27T11:37:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707519513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707519513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707519513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707519513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707519513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2, entries=150, sequenceid=16, filesize=30.2 K 2024-11-27T11:37:39,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/c80027be68624da0a20732766c7d7e3f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/c80027be68624da0a20732766c7d7e3f 2024-11-27T11:37:39,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/c80027be68624da0a20732766c7d7e3f, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T11:37:39,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/6bf4c6e929ce4429b43a291388aae5a7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6bf4c6e929ce4429b43a291388aae5a7 2024-11-27T11:37:39,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6bf4c6e929ce4429b43a291388aae5a7, entries=150, sequenceid=16, filesize=11.7 K 2024-11-27T11:37:39,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 00f69fd5a9af317bd8000447a4cf50b8 in 148ms, sequenceid=16, compaction requested=false 2024-11-27T11:37:39,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:39,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-27T11:37:39,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:39,528 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:37:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112740da0b5e90364b13bd9490ad8242225a_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707459404/Put/seqid=0 2024-11-27T11:37:39,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742314_1490 (size=12154) 2024-11-27T11:37:39,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-27T11:37:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:39,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707519730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,734 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707519731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707519732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707519733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707519733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707519842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707519842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,845 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707519842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707519842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707519842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:39,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:39,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:39,942 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112740da0b5e90364b13bd9490ad8242225a_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740da0b5e90364b13bd9490ad8242225a_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:39,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/bd114a06ca7e4922a97cb2d25394a6d5, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:39,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/bd114a06ca7e4922a97cb2d25394a6d5 is 175, key is test_row_0/A:col10/1732707459404/Put/seqid=0 2024-11-27T11:37:39,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742315_1491 (size=30955) 2024-11-27T11:37:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-27T11:37:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707520052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707520052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707520052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707520053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707520053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,348 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/bd114a06ca7e4922a97cb2d25394a6d5 2024-11-27T11:37:40,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/010d2c5557bd47d8813b632829920d93 is 50, key is test_row_0/B:col10/1732707459404/Put/seqid=0 2024-11-27T11:37:40,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742316_1492 (size=12001) 2024-11-27T11:37:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707520363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707520363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707520363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707520363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707520363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-27T11:37:40,759 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/010d2c5557bd47d8813b632829920d93 2024-11-27T11:37:40,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/1bb07cb83fc4495f98070873557c7072 is 50, key is test_row_0/C:col10/1732707459404/Put/seqid=0 2024-11-27T11:37:40,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742317_1493 (size=12001) 2024-11-27T11:37:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707520873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707520873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707520873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707520873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707520873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:40,966 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:37:41,171 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/1bb07cb83fc4495f98070873557c7072 2024-11-27T11:37:41,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/bd114a06ca7e4922a97cb2d25394a6d5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5 2024-11-27T11:37:41,178 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5, entries=150, sequenceid=40, filesize=30.2 K 2024-11-27T11:37:41,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/010d2c5557bd47d8813b632829920d93 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/010d2c5557bd47d8813b632829920d93 2024-11-27T11:37:41,183 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/010d2c5557bd47d8813b632829920d93, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:37:41,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/1bb07cb83fc4495f98070873557c7072 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/1bb07cb83fc4495f98070873557c7072 2024-11-27T11:37:41,187 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/1bb07cb83fc4495f98070873557c7072, entries=150, sequenceid=40, filesize=11.7 K 2024-11-27T11:37:41,188 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 00f69fd5a9af317bd8000447a4cf50b8 in 1659ms, sequenceid=40, compaction requested=false 2024-11-27T11:37:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:41,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-27T11:37:41,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-27T11:37:41,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-27T11:37:41,191 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-27T11:37:41,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.8240 sec 2024-11-27T11:37:41,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-27T11:37:41,523 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:41,523 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:41,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-27T11:37:41,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T11:37:41,525 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:41,526 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:41,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T11:37:41,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-27T11:37:41,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:41,678 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-27T11:37:41,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270a947f0639974799b0dccbddbd4a5954_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707459732/Put/seqid=0 2024-11-27T11:37:41,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742318_1494 (size=12154) 2024-11-27T11:37:41,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T11:37:41,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:41,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:41,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:41,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707521907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,912 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:41,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707521909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:41,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707521911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,915 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,916 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:41,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:41,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707521911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707521912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707522022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707522022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,023 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707522022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707522023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707522023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,026 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:42,093 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270a947f0639974799b0dccbddbd4a5954_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270a947f0639974799b0dccbddbd4a5954_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ef7b99d98f6041258bff5f853599d807, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:42,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ef7b99d98f6041258bff5f853599d807 is 175, key is test_row_0/A:col10/1732707459732/Put/seqid=0 2024-11-27T11:37:42,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742319_1495 (size=30955) 2024-11-27T11:37:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T11:37:42,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707522232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707522232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707522232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,234 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707522233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707522233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,235 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,498 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ef7b99d98f6041258bff5f853599d807 2024-11-27T11:37:42,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/8ccd33f10e7b470d8f33abf061b37729 is 50, key is test_row_0/B:col10/1732707459732/Put/seqid=0 2024-11-27T11:37:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742320_1496 (size=12001) 2024-11-27T11:37:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707522542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707522542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707522543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707522542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707522543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:42,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:42,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T11:37:42,915 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/8ccd33f10e7b470d8f33abf061b37729 2024-11-27T11:37:42,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/6b0541bed29c4e818cbf14cdd4fd993f is 50, key is test_row_0/C:col10/1732707459732/Put/seqid=0 2024-11-27T11:37:42,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742321_1497 (size=12001) 2024-11-27T11:37:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707523053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707523053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:43,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707523053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707523053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:43,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:43,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707523053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:43,326 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/6b0541bed29c4e818cbf14cdd4fd993f 2024-11-27T11:37:43,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ef7b99d98f6041258bff5f853599d807 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807 2024-11-27T11:37:43,333 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807, entries=150, sequenceid=53, filesize=30.2 K 2024-11-27T11:37:43,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/8ccd33f10e7b470d8f33abf061b37729 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/8ccd33f10e7b470d8f33abf061b37729 2024-11-27T11:37:43,337 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/8ccd33f10e7b470d8f33abf061b37729, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T11:37:43,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/6b0541bed29c4e818cbf14cdd4fd993f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6b0541bed29c4e818cbf14cdd4fd993f 2024-11-27T11:37:43,342 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6b0541bed29c4e818cbf14cdd4fd993f, entries=150, sequenceid=53, filesize=11.7 K 2024-11-27T11:37:43,343 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 00f69fd5a9af317bd8000447a4cf50b8 in 1664ms, sequenceid=53, compaction requested=true 2024-11-27T11:37:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:43,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-27T11:37:43,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-27T11:37:43,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-27T11:37:43,346 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8180 sec 2024-11-27T11:37:43,348 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.8240 sec 2024-11-27T11:37:43,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-27T11:37:43,673 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:43,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:43,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-27T11:37:43,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-27T11:37:43,675 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:43,676 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:43,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:43,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-27T11:37:43,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-27T11:37:43,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:43,828 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-27T11:37:43,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bbc397ead2424d5ba35683fa21ba39dd_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707461902/Put/seqid=0 2024-11-27T11:37:43,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742322_1498 (size=12154) 2024-11-27T11:37:43,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-27T11:37:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:44,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:44,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707524077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707524078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707524078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707524079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707524081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,086 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707524192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707524192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,196 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707524192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707524193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707524193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,197 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:44,245 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127bbc397ead2424d5ba35683fa21ba39dd_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bbc397ead2424d5ba35683fa21ba39dd_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/95f882071bc54dc29924f2f24b34ab82, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:44,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/95f882071bc54dc29924f2f24b34ab82 is 175, key is test_row_0/A:col10/1732707461902/Put/seqid=0 2024-11-27T11:37:44,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742323_1499 (size=30955) 2024-11-27T11:37:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-27T11:37:44,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707524402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707524403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707524403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707524403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707524403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,650 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/95f882071bc54dc29924f2f24b34ab82 2024-11-27T11:37:44,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/64b98752554343c1a2ae8c144f53ae94 is 50, key is test_row_0/B:col10/1732707461902/Put/seqid=0 2024-11-27T11:37:44,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742324_1500 (size=12001) 2024-11-27T11:37:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707524712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707524712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707524712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707524713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707524713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:44,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:44,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-27T11:37:45,062 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/64b98752554343c1a2ae8c144f53ae94 2024-11-27T11:37:45,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/4786a357077d4315baea8f58658b37d6 is 50, key is test_row_0/C:col10/1732707461902/Put/seqid=0 2024-11-27T11:37:45,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742325_1501 (size=12001) 2024-11-27T11:37:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:45,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707525223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:45,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707525223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707525223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707525223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:45,225 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707525223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,225 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:45,226 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:45,475 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/4786a357077d4315baea8f58658b37d6 2024-11-27T11:37:45,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/95f882071bc54dc29924f2f24b34ab82 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82 2024-11-27T11:37:45,482 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82, entries=150, sequenceid=77, filesize=30.2 K 2024-11-27T11:37:45,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/64b98752554343c1a2ae8c144f53ae94 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/64b98752554343c1a2ae8c144f53ae94 2024-11-27T11:37:45,486 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/64b98752554343c1a2ae8c144f53ae94, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T11:37:45,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/4786a357077d4315baea8f58658b37d6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/4786a357077d4315baea8f58658b37d6 2024-11-27T11:37:45,490 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/4786a357077d4315baea8f58658b37d6, entries=150, sequenceid=77, filesize=11.7 K 2024-11-27T11:37:45,491 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 00f69fd5a9af317bd8000447a4cf50b8 in 1663ms, sequenceid=77, compaction requested=true 2024-11-27T11:37:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-27T11:37:45,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-27T11:37:45,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-27T11:37:45,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8170 sec 2024-11-27T11:37:45,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.8220 sec 2024-11-27T11:37:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-27T11:37:45,823 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:45,823 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-27T11:37:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T11:37:45,825 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:45,826 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:45,826 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T11:37:45,978 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-27T11:37:45,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:45,978 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:37:45,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a98e5e9259704308b51fa41890702e32_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707464068/Put/seqid=0 2024-11-27T11:37:45,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742326_1502 (size=12154) 2024-11-27T11:37:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T11:37:46,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:46,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707526285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707526285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707526285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707526286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707526286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,291 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:46,393 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a98e5e9259704308b51fa41890702e32_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a98e5e9259704308b51fa41890702e32_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:46,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/59dd21cf09f24a3a9c81919f94f8ee33, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:46,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/59dd21cf09f24a3a9c81919f94f8ee33 is 175, key is test_row_0/A:col10/1732707464068/Put/seqid=0 2024-11-27T11:37:46,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707526392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707526392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,397 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742327_1503 (size=30955) 2024-11-27T11:37:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707526402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707526403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707526403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,406 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T11:37:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707526602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707526602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707526613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707526613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707526613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,798 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/59dd21cf09f24a3a9c81919f94f8ee33 2024-11-27T11:37:46,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/769ba05d32f84e5382516223cd260c7a is 50, key is test_row_0/B:col10/1732707464068/Put/seqid=0 2024-11-27T11:37:46,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742328_1504 (size=12001) 2024-11-27T11:37:46,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707526912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707526912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707526922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,926 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707526923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707526923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:46,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:46,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T11:37:47,208 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/769ba05d32f84e5382516223cd260c7a 2024-11-27T11:37:47,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/d538bac0183a4b8d8e8caf9bb26db66f is 50, key is test_row_0/C:col10/1732707464068/Put/seqid=0 2024-11-27T11:37:47,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742329_1505 (size=12001) 2024-11-27T11:37:47,219 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/d538bac0183a4b8d8e8caf9bb26db66f 2024-11-27T11:37:47,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/59dd21cf09f24a3a9c81919f94f8ee33 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33 2024-11-27T11:37:47,226 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33, entries=150, sequenceid=89, filesize=30.2 K 2024-11-27T11:37:47,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/769ba05d32f84e5382516223cd260c7a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/769ba05d32f84e5382516223cd260c7a 2024-11-27T11:37:47,230 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/769ba05d32f84e5382516223cd260c7a, entries=150, sequenceid=89, filesize=11.7 K 2024-11-27T11:37:47,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/d538bac0183a4b8d8e8caf9bb26db66f as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/d538bac0183a4b8d8e8caf9bb26db66f 2024-11-27T11:37:47,239 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/d538bac0183a4b8d8e8caf9bb26db66f, entries=150, sequenceid=89, filesize=11.7 K 2024-11-27T11:37:47,239 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 00f69fd5a9af317bd8000447a4cf50b8 in 1261ms, sequenceid=89, compaction requested=true 2024-11-27T11:37:47,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:47,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-27T11:37:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-27T11:37:47,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-27T11:37:47,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4150 sec 2024-11-27T11:37:47,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4200 sec 2024-11-27T11:37:47,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:47,425 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:47,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:37:47,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dda0d40780a94e908cc76aa78b11c9f2_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:47,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742330_1506 (size=17034) 2024-11-27T11:37:47,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:47,439 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127dda0d40780a94e908cc76aa78b11c9f2_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dda0d40780a94e908cc76aa78b11c9f2_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:47,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/6b752013c28442539e8325325233c326, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:47,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707527436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/6b752013c28442539e8325325233c326 is 175, key is test_row_0/A:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742331_1507 (size=48139) 2024-11-27T11:37:47,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707527437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707527439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707527440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707527440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,444 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,445 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707527542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707527552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707527552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707527552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707527552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,753 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707527752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707527762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707527762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707527762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707527763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:47,843 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/6b752013c28442539e8325325233c326 2024-11-27T11:37:47,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/cbd870306cf54350bb885f552255c52a is 50, key is test_row_0/B:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:47,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742332_1508 (size=12001) 2024-11-27T11:37:47,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/cbd870306cf54350bb885f552255c52a 2024-11-27T11:37:47,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/b38eb344096b42a09c7137a4aa39cb21 is 50, key is test_row_0/C:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:47,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742333_1509 (size=12001) 2024-11-27T11:37:47,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-27T11:37:47,972 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:47,973 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-27T11:37:47,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T11:37:47,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:47,975 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:47,975 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:48,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707528062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707528073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707528073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707528073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707528073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T11:37:48,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-27T11:37:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:48,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:48,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:48,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/b38eb344096b42a09c7137a4aa39cb21 2024-11-27T11:37:48,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/6b752013c28442539e8325325233c326 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326 2024-11-27T11:37:48,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326, entries=250, sequenceid=116, filesize=47.0 K 2024-11-27T11:37:48,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/cbd870306cf54350bb885f552255c52a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/cbd870306cf54350bb885f552255c52a 2024-11-27T11:37:48,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-27T11:37:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:48,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/cbd870306cf54350bb885f552255c52a, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T11:37:48,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:48,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:48,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/b38eb344096b42a09c7137a4aa39cb21 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b38eb344096b42a09c7137a4aa39cb21 2024-11-27T11:37:48,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b38eb344096b42a09c7137a4aa39cb21, entries=150, sequenceid=116, filesize=11.7 K 2024-11-27T11:37:48,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 00f69fd5a9af317bd8000447a4cf50b8 in 861ms, sequenceid=116, compaction requested=true 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:48,286 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:48,286 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T11:37:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 202914 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/B is initiating minor compaction (all files) 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/A is initiating minor compaction (all files) 2024-11-27T11:37:48,288 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/B in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,288 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/A in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,288 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/c80027be68624da0a20732766c7d7e3f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/010d2c5557bd47d8813b632829920d93, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/8ccd33f10e7b470d8f33abf061b37729, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/64b98752554343c1a2ae8c144f53ae94, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/769ba05d32f84e5382516223cd260c7a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/cbd870306cf54350bb885f552255c52a] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=70.3 K 2024-11-27T11:37:48,288 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=198.2 K 2024-11-27T11:37:48,288 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326] 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c80027be68624da0a20732766c7d7e3f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732707459374 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cf9976355844aab9ca456b4b58f0ca2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732707459374 2024-11-27T11:37:48,288 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 010d2c5557bd47d8813b632829920d93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707459402 2024-11-27T11:37:48,289 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd114a06ca7e4922a97cb2d25394a6d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707459402 2024-11-27T11:37:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ccd33f10e7b470d8f33abf061b37729, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707459725 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef7b99d98f6041258bff5f853599d807, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707459725 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 64b98752554343c1a2ae8c144f53ae94, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707461902 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95f882071bc54dc29924f2f24b34ab82, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707461902 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 769ba05d32f84e5382516223cd260c7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732707464068 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59dd21cf09f24a3a9c81919f94f8ee33, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732707464068 2024-11-27T11:37:48,298 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting cbd870306cf54350bb885f552255c52a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707466284 2024-11-27T11:37:48,299 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b752013c28442539e8325325233c326, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707466263 2024-11-27T11:37:48,307 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:48,310 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#B#compaction#430 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:48,310 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/b222685bad3d47e6bbda6e4bf16d77fb is 50, key is test_row_0/B:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:48,311 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112715bffffbb299407688b4ae3add6f44d7_00f69fd5a9af317bd8000447a4cf50b8 store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:48,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742334_1510 (size=12207) 2024-11-27T11:37:48,315 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112715bffffbb299407688b4ae3add6f44d7_00f69fd5a9af317bd8000447a4cf50b8, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:48,315 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112715bffffbb299407688b4ae3add6f44d7_00f69fd5a9af317bd8000447a4cf50b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:48,319 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/b222685bad3d47e6bbda6e4bf16d77fb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b222685bad3d47e6bbda6e4bf16d77fb 2024-11-27T11:37:48,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742335_1511 (size=4469) 2024-11-27T11:37:48,323 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/B of 00f69fd5a9af317bd8000447a4cf50b8 into b222685bad3d47e6bbda6e4bf16d77fb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:48,323 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:48,323 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/B, priority=10, startTime=1732707468286; duration=0sec 2024-11-27T11:37:48,323 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:48,323 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:B 2024-11-27T11:37:48,323 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-27T11:37:48,325 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72006 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-27T11:37:48,325 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/C is initiating minor compaction (all files) 2024-11-27T11:37:48,325 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/C in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,325 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6bf4c6e929ce4429b43a291388aae5a7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/1bb07cb83fc4495f98070873557c7072, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6b0541bed29c4e818cbf14cdd4fd993f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/4786a357077d4315baea8f58658b37d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/d538bac0183a4b8d8e8caf9bb26db66f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b38eb344096b42a09c7137a4aa39cb21] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=70.3 K 2024-11-27T11:37:48,325 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf4c6e929ce4429b43a291388aae5a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732707459374 2024-11-27T11:37:48,325 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bb07cb83fc4495f98070873557c7072, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732707459402 2024-11-27T11:37:48,325 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b0541bed29c4e818cbf14cdd4fd993f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732707459725 2024-11-27T11:37:48,326 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 4786a357077d4315baea8f58658b37d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732707461902 2024-11-27T11:37:48,326 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting d538bac0183a4b8d8e8caf9bb26db66f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732707464068 2024-11-27T11:37:48,326 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b38eb344096b42a09c7137a4aa39cb21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707466284 2024-11-27T11:37:48,335 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:48,335 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/79ae9e5435dd46a799d275e0f0ffdef3 is 50, key is test_row_0/C:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:48,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742336_1512 (size=12207) 2024-11-27T11:37:48,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-27T11:37:48,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:48,433 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-27T11:37:48,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279c7fdc79b1a04f82b2bdef298213512d_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707467439/Put/seqid=0 2024-11-27T11:37:48,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742337_1513 (size=12154) 2024-11-27T11:37:48,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:48,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:48,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T11:37:48,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707528604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707528606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707528607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,612 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707528608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,613 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707528610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,614 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707528712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,716 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,720 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#A#compaction#431 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:48,720 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/5da951d9a11c42878291f6d8bab38426 is 175, key is test_row_0/A:col10/1732707467425/Put/seqid=0 2024-11-27T11:37:48,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742338_1514 (size=31161) 2024-11-27T11:37:48,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707528723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707528723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707528723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707528723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,727 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,749 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/79ae9e5435dd46a799d275e0f0ffdef3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/79ae9e5435dd46a799d275e0f0ffdef3 2024-11-27T11:37:48,752 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/C of 00f69fd5a9af317bd8000447a4cf50b8 into 79ae9e5435dd46a799d275e0f0ffdef3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:48,753 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:48,753 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/C, priority=10, startTime=1732707468286; duration=0sec 2024-11-27T11:37:48,753 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:48,753 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:C 2024-11-27T11:37:48,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:48,848 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279c7fdc79b1a04f82b2bdef298213512d_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279c7fdc79b1a04f82b2bdef298213512d_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ba5f9a4ccfbd4c919f60ca72ec52b963, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ba5f9a4ccfbd4c919f60ca72ec52b963 is 175, key is test_row_0/A:col10/1732707467439/Put/seqid=0 2024-11-27T11:37:48,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742339_1515 (size=30955) 2024-11-27T11:37:48,853 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ba5f9a4ccfbd4c919f60ca72ec52b963 2024-11-27T11:37:48,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/0c07b12ed28f4ea39371ef18039ca6d4 is 50, key is test_row_0/B:col10/1732707467439/Put/seqid=0 2024-11-27T11:37:48,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742340_1516 (size=12001) 2024-11-27T11:37:48,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707528923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707528933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707528933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707528933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:48,936 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707528933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:48,937 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T11:37:49,129 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/5da951d9a11c42878291f6d8bab38426 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/5da951d9a11c42878291f6d8bab38426 2024-11-27T11:37:49,136 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/A of 00f69fd5a9af317bd8000447a4cf50b8 into 5da951d9a11c42878291f6d8bab38426(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:49,136 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:49,136 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/A, priority=10, startTime=1732707468286; duration=0sec 2024-11-27T11:37:49,137 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:49,137 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:A 2024-11-27T11:37:49,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707529233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,233 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707529243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707529243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707529243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707529243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,266 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/0c07b12ed28f4ea39371ef18039ca6d4 2024-11-27T11:37:49,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/cf8f9400dc464ef1aa675cecdf33b3aa is 50, key is test_row_0/C:col10/1732707467439/Put/seqid=0 2024-11-27T11:37:49,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742341_1517 (size=12001) 2024-11-27T11:37:49,276 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/cf8f9400dc464ef1aa675cecdf33b3aa 2024-11-27T11:37:49,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/ba5f9a4ccfbd4c919f60ca72ec52b963 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963 2024-11-27T11:37:49,284 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963, entries=150, sequenceid=127, filesize=30.2 K 2024-11-27T11:37:49,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/0c07b12ed28f4ea39371ef18039ca6d4 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/0c07b12ed28f4ea39371ef18039ca6d4 2024-11-27T11:37:49,288 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/0c07b12ed28f4ea39371ef18039ca6d4, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T11:37:49,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/cf8f9400dc464ef1aa675cecdf33b3aa as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/cf8f9400dc464ef1aa675cecdf33b3aa 2024-11-27T11:37:49,291 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/cf8f9400dc464ef1aa675cecdf33b3aa, entries=150, sequenceid=127, filesize=11.7 K 2024-11-27T11:37:49,292 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 00f69fd5a9af317bd8000447a4cf50b8 in 859ms, sequenceid=127, compaction requested=false 2024-11-27T11:37:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:49,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-27T11:37:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-27T11:37:49,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-27T11:37:49,295 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3180 sec 2024-11-27T11:37:49,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.3230 sec 2024-11-27T11:37:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:49,745 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:49,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-27T11:37:49,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270d37367127364bad80c061ff1466193e_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:49,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742342_1518 (size=12304) 2024-11-27T11:37:49,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707529753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,757 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707529755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707529756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707529757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707529757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707529862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707529862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707529863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707529863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707529863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:49,865 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707530072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707530072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707530072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,075 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707530073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707530073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,076 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-27T11:37:50,122 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:50,123 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:50,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-27T11:37:50,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T11:37:50,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:50,125 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:50,126 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:50,156 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:50,159 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411270d37367127364bad80c061ff1466193e_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270d37367127364bad80c061ff1466193e_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:50,159 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/22d9fa6cbd5b4dbd9365d420b4913ca5, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:50,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/22d9fa6cbd5b4dbd9365d420b4913ca5 is 175, key is test_row_0/A:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:50,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742343_1519 (size=31105) 2024-11-27T11:37:50,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T11:37:50,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T11:37:50,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:50,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,278 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707530382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707530382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707530383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707530383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707530383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,386 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T11:37:50,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:50,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T11:37:50,564 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/22d9fa6cbd5b4dbd9365d420b4913ca5 2024-11-27T11:37:50,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/b45b430f492546d0af1418f24539f427 is 50, key is test_row_0/B:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:50,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742344_1520 (size=12151) 2024-11-27T11:37:50,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T11:37:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T11:37:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:50,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T11:37:50,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T11:37:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:50,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:50,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707530892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707530893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707530893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707530893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707530893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:50,895 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:50,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:50,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/b45b430f492546d0af1418f24539f427 2024-11-27T11:37:50,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/8f32585098404fb9be9d62b25c60c962 is 50, key is test_row_0/C:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:50,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742345_1521 (size=12151) 2024-11-27T11:37:50,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/8f32585098404fb9be9d62b25c60c962 2024-11-27T11:37:51,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/22d9fa6cbd5b4dbd9365d420b4913ca5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5 2024-11-27T11:37:51,011 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5, entries=150, sequenceid=157, filesize=30.4 K 2024-11-27T11:37:51,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/b45b430f492546d0af1418f24539f427 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b45b430f492546d0af1418f24539f427 2024-11-27T11:37:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b45b430f492546d0af1418f24539f427, entries=150, sequenceid=157, filesize=11.9 K 2024-11-27T11:37:51,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/8f32585098404fb9be9d62b25c60c962 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/8f32585098404fb9be9d62b25c60c962 2024-11-27T11:37:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/8f32585098404fb9be9d62b25c60c962, entries=150, sequenceid=157, filesize=11.9 K 2024-11-27T11:37:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 00f69fd5a9af317bd8000447a4cf50b8 in 1275ms, sequenceid=157, compaction requested=true 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:51,021 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:51,021 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,022 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93221 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:51,022 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:51,022 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/A is initiating minor compaction (all files) 2024-11-27T11:37:51,022 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/B is initiating minor compaction (all files) 2024-11-27T11:37:51,022 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/A in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:51,022 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/B in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:51,022 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b222685bad3d47e6bbda6e4bf16d77fb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/0c07b12ed28f4ea39371ef18039ca6d4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b45b430f492546d0af1418f24539f427] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=35.5 K 2024-11-27T11:37:51,022 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/5da951d9a11c42878291f6d8bab38426, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=91.0 K 2024-11-27T11:37:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,022 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:51,022 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/5da951d9a11c42878291f6d8bab38426, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5] 2024-11-27T11:37:51,022 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b222685bad3d47e6bbda6e4bf16d77fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707466284 2024-11-27T11:37:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,023 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c07b12ed28f4ea39371ef18039ca6d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732707467430 2024-11-27T11:37:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,023 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b45b430f492546d0af1418f24539f427, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732707468607 2024-11-27T11:37:51,023 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5da951d9a11c42878291f6d8bab38426, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707466284 2024-11-27T11:37:51,023 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba5f9a4ccfbd4c919f60ca72ec52b963, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732707467430 2024-11-27T11:37:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,024 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22d9fa6cbd5b4dbd9365d420b4913ca5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732707468607 2024-11-27T11:37:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-27T11:37:51,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:51,042 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-27T11:37:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,045 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#B#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:51,045 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/f5bffc26aeff4612905cf26bf5025bb3 is 50, key is test_row_0/B:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,049 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,054 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411273a3bc3a74d4247aab69eaa36860d5001_00f69fd5a9af317bd8000447a4cf50b8 store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:51,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273b74ca87ff2f4d93986fadbb1bcd9dbb_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_1/A:col10/1732707469755/Put/seqid=0 2024-11-27T11:37:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,056 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411273a3bc3a74d4247aab69eaa36860d5001_00f69fd5a9af317bd8000447a4cf50b8, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:51,056 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273a3bc3a74d4247aab69eaa36860d5001_00f69fd5a9af317bd8000447a4cf50b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742346_1522 (size=12459) 2024-11-27T11:37:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,062 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/f5bffc26aeff4612905cf26bf5025bb3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f5bffc26aeff4612905cf26bf5025bb3 2024-11-27T11:37:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,067 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/B of 00f69fd5a9af317bd8000447a4cf50b8 into f5bffc26aeff4612905cf26bf5025bb3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:51,067 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/B, priority=13, startTime=1732707471021; duration=0sec 2024-11-27T11:37:51,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:51,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:B 2024-11-27T11:37:51,067 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:51,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,069 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:51,069 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/C is initiating minor compaction (all files) 2024-11-27T11:37:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,069 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/C in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:51,069 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/79ae9e5435dd46a799d275e0f0ffdef3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/cf8f9400dc464ef1aa675cecdf33b3aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/8f32585098404fb9be9d62b25c60c962] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=35.5 K 2024-11-27T11:37:51,069 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 79ae9e5435dd46a799d275e0f0ffdef3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732707466284 2024-11-27T11:37:51,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,070 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting cf8f9400dc464ef1aa675cecdf33b3aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732707467430 2024-11-27T11:37:51,070 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f32585098404fb9be9d62b25c60c962, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732707468607 2024-11-27T11:37:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742347_1523 (size=9814) 2024-11-27T11:37:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742348_1524 (size=4469) 2024-11-27T11:37:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,075 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#A#compaction#440 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,075 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/b96bfdefee0849ed86e6cecaf638fdd7 is 175, key is test_row_0/A:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,078 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#C#compaction#442 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,078 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/ffd642656cd0455182565a2cf1f0b2ff is 50, key is test_row_0/C:col10/1732707468608/Put/seqid=0 2024-11-27T11:37:51,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742349_1525 (size=31413) 2024-11-27T11:37:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742350_1526 (size=12459) 2024-11-27T11:37:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,089 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/b96bfdefee0849ed86e6cecaf638fdd7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/b96bfdefee0849ed86e6cecaf638fdd7 2024-11-27T11:37:51,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,092 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/ffd642656cd0455182565a2cf1f0b2ff as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/ffd642656cd0455182565a2cf1f0b2ff 2024-11-27T11:37:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,094 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/A of 00f69fd5a9af317bd8000447a4cf50b8 into b96bfdefee0849ed86e6cecaf638fdd7(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:51,094 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:51,094 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/A, priority=13, startTime=1732707471021; duration=0sec 2024-11-27T11:37:51,094 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:51,094 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:A 2024-11-27T11:37:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,096 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/C of 00f69fd5a9af317bd8000447a4cf50b8 into ffd642656cd0455182565a2cf1f0b2ff(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:51,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,096 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:51,096 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/C, priority=13, startTime=1732707471021; duration=0sec 2024-11-27T11:37:51,097 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:51,097 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:C 2024-11-27T11:37:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T11:37:51,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,478 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273b74ca87ff2f4d93986fadbb1bcd9dbb_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273b74ca87ff2f4d93986fadbb1bcd9dbb_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/2e57e9b9780a4a0781140057b122e6bb, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/2e57e9b9780a4a0781140057b122e6bb is 175, key is test_row_1/A:col10/1732707469755/Put/seqid=0 2024-11-27T11:37:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742351_1527 (size=22461) 2024-11-27T11:37:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,888 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=166, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/2e57e9b9780a4a0781140057b122e6bb 2024-11-27T11:37:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/9421e5a1ffdf42569533cf614eaea77a is 50, key is test_row_1/B:col10/1732707469755/Put/seqid=0 2024-11-27T11:37:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742352_1528 (size=9757) 2024-11-27T11:37:51,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,902 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/9421e5a1ffdf42569533cf614eaea77a 2024-11-27T11:37:51,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:51,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:51,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/0506b6e1123b4612a3a78d15954873bb is 50, key is test_row_1/C:col10/1732707469755/Put/seqid=0 2024-11-27T11:37:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742353_1529 (size=9757) 2024-11-27T11:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,922 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/0506b6e1123b4612a3a78d15954873bb 2024-11-27T11:37:51,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/2e57e9b9780a4a0781140057b122e6bb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb 2024-11-27T11:37:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,933 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb, entries=100, sequenceid=166, filesize=21.9 K 2024-11-27T11:37:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/9421e5a1ffdf42569533cf614eaea77a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9421e5a1ffdf42569533cf614eaea77a 2024-11-27T11:37:51,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:51,938 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9421e5a1ffdf42569533cf614eaea77a, entries=100, sequenceid=166, filesize=9.5 K 2024-11-27T11:37:51,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/0506b6e1123b4612a3a78d15954873bb as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0506b6e1123b4612a3a78d15954873bb 2024-11-27T11:37:51,945 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0506b6e1123b4612a3a78d15954873bb, entries=100, sequenceid=166, filesize=9.5 K 2024-11-27T11:37:51,946 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=127.47 KB/130530 for 00f69fd5a9af317bd8000447a4cf50b8 in 904ms, sequenceid=166, compaction requested=false 2024-11-27T11:37:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:51,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-27T11:37:51,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-27T11:37:51,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:51,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:51,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-27T11:37:51,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-27T11:37:51,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8220 sec 2024-11-27T11:37:51,952 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8280 sec 2024-11-27T11:37:51,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a23c877a863748178ede456047485b9e_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:51,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:51,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707531956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742354_1530 (size=17284) 2024-11-27T11:37:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707531957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707531958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707531959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:51,965 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:51,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707531960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707532062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,065 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707532072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707532072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707532073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707532073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707532272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,275 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-27T11:37:52,283 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:52,283 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-27T11:37:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707532283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,285 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707532283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707532283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707532283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,286 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:52,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,363 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:52,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127a23c877a863748178ede456047485b9e_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a23c877a863748178ede456047485b9e_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:52,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e6282f198e8241b4a5796448e597b567, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:52,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e6282f198e8241b4a5796448e597b567 is 175, key is test_row_0/A:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:52,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742355_1531 (size=48389) 2024-11-27T11:37:52,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e6282f198e8241b4a5796448e597b567 2024-11-27T11:37:52,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/add525ba7d934d019cba09ad3bea6cd1 is 50, key is test_row_0/B:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:52,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742356_1532 (size=12151) 2024-11-27T11:37:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:52,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:52,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707532582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,591 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:52,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707532592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,593 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707532593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707532593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707532593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,595 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:52,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:52,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:52,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,746 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/add525ba7d934d019cba09ad3bea6cd1 2024-11-27T11:37:52,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/dc19a5eec7ae4f22b9c4448abe6aab71 is 50, key is test_row_0/C:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:52,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742357_1533 (size=12151) 2024-11-27T11:37:52,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:52,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:52,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:53,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:53,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707533103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:53,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707533103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:53,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707533103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:53,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707533103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:53,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707533103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:53,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:53,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/dc19a5eec7ae4f22b9c4448abe6aab71 2024-11-27T11:37:53,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e6282f198e8241b4a5796448e597b567 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567 2024-11-27T11:37:53,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567, entries=250, sequenceid=194, filesize=47.3 K 2024-11-27T11:37:53,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/add525ba7d934d019cba09ad3bea6cd1 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/add525ba7d934d019cba09ad3bea6cd1 2024-11-27T11:37:53,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:53,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:53,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/add525ba7d934d019cba09ad3bea6cd1, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T11:37:53,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:53,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:53,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/dc19a5eec7ae4f22b9c4448abe6aab71 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/dc19a5eec7ae4f22b9c4448abe6aab71 2024-11-27T11:37:53,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:53,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/dc19a5eec7ae4f22b9c4448abe6aab71, entries=150, sequenceid=194, filesize=11.9 K 2024-11-27T11:37:53,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 00f69fd5a9af317bd8000447a4cf50b8 in 1262ms, sequenceid=194, compaction requested=true 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:53,210 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:53,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:53,210 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/A is initiating minor compaction (all files) 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/B is initiating minor compaction (all files) 2024-11-27T11:37:53,211 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/A in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,211 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/B in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,211 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/b96bfdefee0849ed86e6cecaf638fdd7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=99.9 K 2024-11-27T11:37:53,211 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f5bffc26aeff4612905cf26bf5025bb3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9421e5a1ffdf42569533cf614eaea77a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/add525ba7d934d019cba09ad3bea6cd1] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=33.6 K 2024-11-27T11:37:53,211 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/b96bfdefee0849ed86e6cecaf638fdd7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567] 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting f5bffc26aeff4612905cf26bf5025bb3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732707468607 2024-11-27T11:37:53,211 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting b96bfdefee0849ed86e6cecaf638fdd7, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732707468607 2024-11-27T11:37:53,212 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9421e5a1ffdf42569533cf614eaea77a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732707469755 2024-11-27T11:37:53,212 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e57e9b9780a4a0781140057b122e6bb, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732707469755 2024-11-27T11:37:53,212 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6282f198e8241b4a5796448e597b567, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732707471941 2024-11-27T11:37:53,212 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting add525ba7d934d019cba09ad3bea6cd1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732707471943 2024-11-27T11:37:53,218 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#B#compaction#448 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:53,218 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/7351074c225a4bd9a649bde7fa5ec816 is 50, key is test_row_0/B:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:53,219 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:53,222 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127864a7be2557c499ba6445ff2857a63be_00f69fd5a9af317bd8000447a4cf50b8 store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:53,224 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127864a7be2557c499ba6445ff2857a63be_00f69fd5a9af317bd8000447a4cf50b8, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:53,224 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127864a7be2557c499ba6445ff2857a63be_00f69fd5a9af317bd8000447a4cf50b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:53,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742358_1534 (size=12561) 2024-11-27T11:37:53,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742359_1535 (size=4469) 2024-11-27T11:37:53,233 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/7351074c225a4bd9a649bde7fa5ec816 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/7351074c225a4bd9a649bde7fa5ec816 2024-11-27T11:37:53,238 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/B of 00f69fd5a9af317bd8000447a4cf50b8 into 7351074c225a4bd9a649bde7fa5ec816(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:53,238 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:53,238 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/B, priority=13, startTime=1732707473210; duration=0sec 2024-11-27T11:37:53,239 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:53,239 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:B 2024-11-27T11:37:53,239 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:53,239 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:53,239 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/C is initiating minor compaction (all files) 2024-11-27T11:37:53,240 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/C in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,240 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/ffd642656cd0455182565a2cf1f0b2ff, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0506b6e1123b4612a3a78d15954873bb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/dc19a5eec7ae4f22b9c4448abe6aab71] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=33.6 K 2024-11-27T11:37:53,240 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting ffd642656cd0455182565a2cf1f0b2ff, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732707468607 2024-11-27T11:37:53,240 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 0506b6e1123b4612a3a78d15954873bb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1732707469755 2024-11-27T11:37:53,241 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting dc19a5eec7ae4f22b9c4448abe6aab71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732707471943 2024-11-27T11:37:53,246 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#C#compaction#450 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:53,247 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/0631a58b988c4adebf1ced025b02b9c2 is 50, key is test_row_0/C:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:53,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742360_1536 (size=12561) 2024-11-27T11:37:53,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-27T11:37:53,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:53,357 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:53,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127df69b41ccdf44dedbb09073f82047e02_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707471957/Put/seqid=0 2024-11-27T11:37:53,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742361_1537 (size=12304) 2024-11-27T11:37:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:53,634 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#A#compaction#449 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:53,634 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/d08e71a7254c4fbda364f2c56fe0005a is 175, key is test_row_0/A:col10/1732707471948/Put/seqid=0 2024-11-27T11:37:53,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742362_1538 (size=31515) 2024-11-27T11:37:53,654 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/0631a58b988c4adebf1ced025b02b9c2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0631a58b988c4adebf1ced025b02b9c2 2024-11-27T11:37:53,659 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/C of 00f69fd5a9af317bd8000447a4cf50b8 into 0631a58b988c4adebf1ced025b02b9c2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:53,659 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:53,659 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/C, priority=13, startTime=1732707473210; duration=0sec 2024-11-27T11:37:53,659 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:53,659 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:C 2024-11-27T11:37:53,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:53,772 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127df69b41ccdf44dedbb09073f82047e02_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127df69b41ccdf44dedbb09073f82047e02_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:53,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e8a2e2d4926149ad950603dd3cea2316, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:53,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e8a2e2d4926149ad950603dd3cea2316 is 175, key is test_row_0/A:col10/1732707471957/Put/seqid=0 2024-11-27T11:37:53,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742363_1539 (size=31105) 2024-11-27T11:37:54,049 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/d08e71a7254c4fbda364f2c56fe0005a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d08e71a7254c4fbda364f2c56fe0005a 2024-11-27T11:37:54,053 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/A of 00f69fd5a9af317bd8000447a4cf50b8 into d08e71a7254c4fbda364f2c56fe0005a(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:54,053 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:54,054 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/A, priority=13, startTime=1732707473210; duration=0sec 2024-11-27T11:37:54,054 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:54,054 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:A 2024-11-27T11:37:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:54,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:54,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707534134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707534134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707534135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,140 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707534136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707534137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,141 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,182 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e8a2e2d4926149ad950603dd3cea2316 2024-11-27T11:37:54,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/3e4d1f4a9cc64e34b30d9ba77be08d50 is 50, key is test_row_0/B:col10/1732707471957/Put/seqid=0 2024-11-27T11:37:54,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742364_1540 (size=12151) 2024-11-27T11:37:54,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707534242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707534242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707534242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,244 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707534252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707534253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707534452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707534452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707534452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,455 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707534463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707534463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,592 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/3e4d1f4a9cc64e34b30d9ba77be08d50 2024-11-27T11:37:54,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/bb28377d12e7436d98dbfaf48ca12790 is 50, key is test_row_0/C:col10/1732707471957/Put/seqid=0 2024-11-27T11:37:54,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742365_1541 (size=12151) 2024-11-27T11:37:54,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707534762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707534762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,764 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707534762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707534772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707534773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:54,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,003 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/bb28377d12e7436d98dbfaf48ca12790 2024-11-27T11:37:55,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e8a2e2d4926149ad950603dd3cea2316 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316 2024-11-27T11:37:55,010 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316, entries=150, sequenceid=206, filesize=30.4 K 2024-11-27T11:37:55,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/3e4d1f4a9cc64e34b30d9ba77be08d50 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/3e4d1f4a9cc64e34b30d9ba77be08d50 2024-11-27T11:37:55,014 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/3e4d1f4a9cc64e34b30d9ba77be08d50, entries=150, sequenceid=206, filesize=11.9 K 2024-11-27T11:37:55,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/bb28377d12e7436d98dbfaf48ca12790 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/bb28377d12e7436d98dbfaf48ca12790 2024-11-27T11:37:55,017 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/bb28377d12e7436d98dbfaf48ca12790, entries=150, sequenceid=206, filesize=11.9 K 2024-11-27T11:37:55,018 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 00f69fd5a9af317bd8000447a4cf50b8 in 1661ms, sequenceid=206, compaction requested=false 2024-11-27T11:37:55,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:55,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:55,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-27T11:37:55,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-27T11:37:55,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-27T11:37:55,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7330 sec 2024-11-27T11:37:55,022 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.7380 sec 2024-11-27T11:37:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:55,275 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:55,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:37:55,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279d5db8092831431fa6b1f870cd47ebd3_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:55,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707535282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707535283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742366_1542 (size=17284) 2024-11-27T11:37:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707535283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707535283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707535285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,288 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707535393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707535393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707535393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,396 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707535603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707535603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707535603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,606 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:55,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279d5db8092831431fa6b1f870cd47ebd3_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279d5db8092831431fa6b1f870cd47ebd3_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:55,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e56c92b94e6a4f7499e643910ab5f34e, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:55,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e56c92b94e6a4f7499e643910ab5f34e is 175, key is test_row_0/A:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:55,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742367_1543 (size=48389) 2024-11-27T11:37:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707535912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707535912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:55,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707535912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:55,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:56,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=236, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e56c92b94e6a4f7499e643910ab5f34e 2024-11-27T11:37:56,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/2ee357d3347f40d0bf22f02f3f9e343b is 50, key is test_row_0/B:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:56,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742368_1544 (size=12151) 2024-11-27T11:37:56,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707536292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:56,295 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 00f69fd5a9af317bd8000447a4cf50b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,296 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:56,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707536302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:56,304 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in 00f69fd5a9af317bd8000447a4cf50b8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:56,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707536423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:56,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707536423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:56,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707536423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:56,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:56,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-27T11:37:56,453 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:56,453 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-27T11:37:56,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T11:37:56,456 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:56,456 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:56,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:56,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/2ee357d3347f40d0bf22f02f3f9e343b 2024-11-27T11:37:56,513 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/96d9ad5b7bfb4bb0bd4dc11dc433650c is 50, key is test_row_0/C:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:56,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742369_1545 (size=12151) 2024-11-27T11:37:56,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/96d9ad5b7bfb4bb0bd4dc11dc433650c 2024-11-27T11:37:56,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/e56c92b94e6a4f7499e643910ab5f34e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e 2024-11-27T11:37:56,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e, entries=250, sequenceid=236, filesize=47.3 K 2024-11-27T11:37:56,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/2ee357d3347f40d0bf22f02f3f9e343b as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/2ee357d3347f40d0bf22f02f3f9e343b 2024-11-27T11:37:56,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/2ee357d3347f40d0bf22f02f3f9e343b, entries=150, sequenceid=236, filesize=11.9 K 2024-11-27T11:37:56,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/96d9ad5b7bfb4bb0bd4dc11dc433650c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d9ad5b7bfb4bb0bd4dc11dc433650c 2024-11-27T11:37:56,531 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d9ad5b7bfb4bb0bd4dc11dc433650c, entries=150, sequenceid=236, filesize=11.9 K 2024-11-27T11:37:56,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 00f69fd5a9af317bd8000447a4cf50b8 in 1257ms, sequenceid=236, compaction requested=true 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:56,532 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:56,532 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:56,532 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:56,533 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:56,533 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:56,533 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/A is initiating minor compaction (all files) 2024-11-27T11:37:56,533 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/B is initiating minor compaction (all files) 2024-11-27T11:37:56,533 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/A in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:56,533 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/B in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:56,533 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/7351074c225a4bd9a649bde7fa5ec816, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/3e4d1f4a9cc64e34b30d9ba77be08d50, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/2ee357d3347f40d0bf22f02f3f9e343b] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=36.0 K 2024-11-27T11:37:56,533 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d08e71a7254c4fbda364f2c56fe0005a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=108.4 K 2024-11-27T11:37:56,533 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:56,534 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d08e71a7254c4fbda364f2c56fe0005a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e] 2024-11-27T11:37:56,534 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 7351074c225a4bd9a649bde7fa5ec816, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732707471943 2024-11-27T11:37:56,534 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d08e71a7254c4fbda364f2c56fe0005a, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732707471943 2024-11-27T11:37:56,534 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e4d1f4a9cc64e34b30d9ba77be08d50, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732707471955 2024-11-27T11:37:56,534 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8a2e2d4926149ad950603dd3cea2316, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732707471955 2024-11-27T11:37:56,535 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ee357d3347f40d0bf22f02f3f9e343b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732707474133 2024-11-27T11:37:56,535 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting e56c92b94e6a4f7499e643910ab5f34e, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732707474133 2024-11-27T11:37:56,548 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:56,549 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:56,550 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/e1fa98240aea4f588c31721030ccf1c6 is 50, key is test_row_0/B:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:56,551 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241127d7fb02cb3a5a4d30a36e82b4713c04bf_00f69fd5a9af317bd8000447a4cf50b8 store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:56,553 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241127d7fb02cb3a5a4d30a36e82b4713c04bf_00f69fd5a9af317bd8000447a4cf50b8, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:56,553 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127d7fb02cb3a5a4d30a36e82b4713c04bf_00f69fd5a9af317bd8000447a4cf50b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:56,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742370_1546 (size=12663) 2024-11-27T11:37:56,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T11:37:56,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742371_1547 (size=4469) 2024-11-27T11:37:56,574 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#A#compaction#458 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:56,574 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/c2d22e616a674daa86186ba666ff606d is 175, key is test_row_0/A:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:56,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742372_1548 (size=31617) 2024-11-27T11:37:56,583 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/c2d22e616a674daa86186ba666ff606d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/c2d22e616a674daa86186ba666ff606d 2024-11-27T11:37:56,589 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/A of 00f69fd5a9af317bd8000447a4cf50b8 into c2d22e616a674daa86186ba666ff606d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:56,589 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:56,589 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/A, priority=13, startTime=1732707476532; duration=0sec 2024-11-27T11:37:56,589 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:56,589 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:A 2024-11-27T11:37:56,589 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:56,590 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:56,590 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/C is initiating minor compaction (all files) 2024-11-27T11:37:56,590 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/C in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:56,590 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0631a58b988c4adebf1ced025b02b9c2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/bb28377d12e7436d98dbfaf48ca12790, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d9ad5b7bfb4bb0bd4dc11dc433650c] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=36.0 K 2024-11-27T11:37:56,591 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0631a58b988c4adebf1ced025b02b9c2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732707471943 2024-11-27T11:37:56,591 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb28377d12e7436d98dbfaf48ca12790, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732707471955 2024-11-27T11:37:56,591 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96d9ad5b7bfb4bb0bd4dc11dc433650c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732707474133 2024-11-27T11:37:56,597 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:56,598 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/53cd01f85a8e4457b945906c6ebf1aa5 is 50, key is test_row_0/C:col10/1732707475275/Put/seqid=0 2024-11-27T11:37:56,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742373_1549 (size=12663) 2024-11-27T11:37:56,607 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/53cd01f85a8e4457b945906c6ebf1aa5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/53cd01f85a8e4457b945906c6ebf1aa5 2024-11-27T11:37:56,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-27T11:37:56,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:56,609 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-27T11:37:56,611 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/C of 00f69fd5a9af317bd8000447a4cf50b8 into 53cd01f85a8e4457b945906c6ebf1aa5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:56,612 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:56,612 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/C, priority=13, startTime=1732707476532; duration=0sec 2024-11-27T11:37:56,612 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:56,612 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:C 2024-11-27T11:37:56,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112777f450ce2570480980a551fed26a9ef4_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707475280/Put/seqid=0 2024-11-27T11:37:56,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742374_1550 (size=12304) 2024-11-27T11:37:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T11:37:56,964 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/e1fa98240aea4f588c31721030ccf1c6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/e1fa98240aea4f588c31721030ccf1c6 2024-11-27T11:37:56,968 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/B of 00f69fd5a9af317bd8000447a4cf50b8 into e1fa98240aea4f588c31721030ccf1c6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:56,968 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:56,968 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/B, priority=13, startTime=1732707476532; duration=0sec 2024-11-27T11:37:56,968 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:56,968 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:B 2024-11-27T11:37:57,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,028 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112777f450ce2570480980a551fed26a9ef4_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112777f450ce2570480980a551fed26a9ef4_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/d8bde88b69774ddf85fb3bf8bdb167db, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:57,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/d8bde88b69774ddf85fb3bf8bdb167db is 175, key is test_row_0/A:col10/1732707475280/Put/seqid=0 2024-11-27T11:37:57,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742375_1551 (size=31105) 2024-11-27T11:37:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/d8bde88b69774ddf85fb3bf8bdb167db 2024-11-27T11:37:57,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/60b439f20e264195aab8b52c7fa35694 is 50, key is test_row_0/B:col10/1732707475280/Put/seqid=0 2024-11-27T11:37:57,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742376_1552 (size=12151) 2024-11-27T11:37:57,046 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/60b439f20e264195aab8b52c7fa35694 2024-11-27T11:37:57,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/b157f4086dad4a06aaad2229befd10fd is 50, key is test_row_0/C:col10/1732707475280/Put/seqid=0 2024-11-27T11:37:57,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742377_1553 (size=12151) 2024-11-27T11:37:57,059 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/b157f4086dad4a06aaad2229befd10fd 2024-11-27T11:37:57,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/d8bde88b69774ddf85fb3bf8bdb167db as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db 2024-11-27T11:37:57,067 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db, entries=150, sequenceid=245, filesize=30.4 K 2024-11-27T11:37:57,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/60b439f20e264195aab8b52c7fa35694 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/60b439f20e264195aab8b52c7fa35694 2024-11-27T11:37:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,070 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/60b439f20e264195aab8b52c7fa35694, entries=150, sequenceid=245, filesize=11.9 K 2024-11-27T11:37:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/b157f4086dad4a06aaad2229befd10fd as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b157f4086dad4a06aaad2229befd10fd 2024-11-27T11:37:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b157f4086dad4a06aaad2229befd10fd, entries=150, sequenceid=245, filesize=11.9 K 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,076 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 00f69fd5a9af317bd8000447a4cf50b8 in 467ms, sequenceid=245, compaction requested=false 2024-11-27T11:37:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-27T11:37:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-27T11:37:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-27T11:37:57,079 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 621 msec 2024-11-27T11:37:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,081 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 626 msec 2024-11-27T11:37:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-27T11:37:57,084 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,084 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-27T11:37:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T11:37:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,086 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,086 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,086 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-27T11:37:57,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:57,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-27T11:37:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-27T11:37:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-27T11:37:57,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-27T11:37:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,246 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 161 msec 2024-11-27T11:37:57,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-27T11:37:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,403 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,403 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-27T11:37:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-27T11:37:57,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T11:37:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,405 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-27T11:37:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,406 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-27T11:37:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-27T11:37:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,433 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-27T11:37:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:57,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,486 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:57,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-27T11:37:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273a8602959b764d70b54194a801f7c122_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T11:37:57,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742378_1554 (size=22418) 2024-11-27T11:37:57,520 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:57,525 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411273a8602959b764d70b54194a801f7c122_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273a8602959b764d70b54194a801f7c122_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:57,526 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/94cc910a73eb42c6998b899591ad79a8, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:57,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/94cc910a73eb42c6998b899591ad79a8 is 175, key is test_row_0/A:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:57,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707537535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742379_1555 (size=65823) 2024-11-27T11:37:57,543 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/94cc910a73eb42c6998b899591ad79a8 2024-11-27T11:37:57,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707537539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,546 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707537541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,547 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/9ec804c2c2554cc79aa9ecc71c8210db is 50, key is test_row_0/B:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:57,559 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T11:37:57,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742380_1556 (size=12301) 2024-11-27T11:37:57,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:57,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707537653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707537653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707537653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,657 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T11:37:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T11:37:57,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707537862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,863 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707537862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707537862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:57,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T11:37:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:57,867 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:57,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-27T11:37:57,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/9ec804c2c2554cc79aa9ecc71c8210db 2024-11-27T11:37:57,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/a5d4643f59d14676828769a229fc1109 is 50, key is test_row_0/C:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:57,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742381_1557 (size=12301) 2024-11-27T11:37:57,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/a5d4643f59d14676828769a229fc1109 2024-11-27T11:37:57,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/94cc910a73eb42c6998b899591ad79a8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8 2024-11-27T11:37:57,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8, entries=350, sequenceid=259, filesize=64.3 K 2024-11-27T11:37:57,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/9ec804c2c2554cc79aa9ecc71c8210db as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9ec804c2c2554cc79aa9ecc71c8210db 2024-11-27T11:37:57,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9ec804c2c2554cc79aa9ecc71c8210db, entries=150, sequenceid=259, filesize=12.0 K 2024-11-27T11:37:57,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/a5d4643f59d14676828769a229fc1109 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/a5d4643f59d14676828769a229fc1109 2024-11-27T11:37:57,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/a5d4643f59d14676828769a229fc1109, entries=150, sequenceid=259, filesize=12.0 K 2024-11-27T11:37:57,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 00f69fd5a9af317bd8000447a4cf50b8 in 503ms, sequenceid=259, compaction requested=true 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:57,989 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:37:57,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:57,989 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:57,990 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:57,990 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/B is initiating minor compaction (all files) 2024-11-27T11:37:57,990 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/B in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,991 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/e1fa98240aea4f588c31721030ccf1c6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/60b439f20e264195aab8b52c7fa35694, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9ec804c2c2554cc79aa9ecc71c8210db] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=36.2 K 2024-11-27T11:37:57,991 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e1fa98240aea4f588c31721030ccf1c6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732707474133 2024-11-27T11:37:57,991 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 60b439f20e264195aab8b52c7fa35694, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732707475280 2024-11-27T11:37:57,991 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ec804c2c2554cc79aa9ecc71c8210db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732707477468 2024-11-27T11:37:57,992 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128545 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:57,992 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/A is initiating minor compaction (all files) 2024-11-27T11:37:57,992 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/A in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,992 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/c2d22e616a674daa86186ba666ff606d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=125.5 K 2024-11-27T11:37:57,992 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:57,992 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/c2d22e616a674daa86186ba666ff606d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8] 2024-11-27T11:37:57,993 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2d22e616a674daa86186ba666ff606d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732707474133 2024-11-27T11:37:57,994 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8bde88b69774ddf85fb3bf8bdb167db, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732707475280 2024-11-27T11:37:58,004 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94cc910a73eb42c6998b899591ad79a8, keycount=350, bloomtype=ROW, size=64.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732707477458 2024-11-27T11:37:58,010 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#B#compaction#466 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:58,011 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/a33898581a9c4b918a21eab53ce420f8 is 50, key is test_row_0/B:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:58,013 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:58,016 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411278d8438143350478c98438f64be9188cf_00f69fd5a9af317bd8000447a4cf50b8 store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:58,018 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411278d8438143350478c98438f64be9188cf_00f69fd5a9af317bd8000447a4cf50b8, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:58,018 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278d8438143350478c98438f64be9188cf_00f69fd5a9af317bd8000447a4cf50b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:58,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=35461 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-27T11:37:58,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:58,020 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-27T11:37:58,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742382_1558 (size=12915) 2024-11-27T11:37:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T11:37:58,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127213389f5ef754a82bfd9a6cedbb3405b_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707477522/Put/seqid=0 2024-11-27T11:37:58,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742383_1559 (size=4469) 2024-11-27T11:37:58,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742384_1560 (size=12454) 2024-11-27T11:37:58,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:58,066 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127213389f5ef754a82bfd9a6cedbb3405b_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127213389f5ef754a82bfd9a6cedbb3405b_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:58,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/a2c2c321db8a4e11a30ae7efdd3f4eb6, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:58,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/a2c2c321db8a4e11a30ae7efdd3f4eb6 is 175, key is test_row_0/A:col10/1732707477522/Put/seqid=0 2024-11-27T11:37:58,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742385_1561 (size=31255) 2024-11-27T11:37:58,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:58,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. as already flushing 2024-11-27T11:37:58,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707538182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707538186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707538187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707538293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707538293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,304 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54620 deadline: 1732707538302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,305 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_2' in 00f69fd5a9af317bd8000447a4cf50b8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:58,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707538302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54642 deadline: 1732707538312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,315 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 5b617c7edc8e,35461,1732707299005 for 'test_row_0' in 00f69fd5a9af317bd8000447a4cf50b8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-27T11:37:58,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,316 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,432 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/a33898581a9c4b918a21eab53ce420f8 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/a33898581a9c4b918a21eab53ce420f8 2024-11-27T11:37:58,436 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/B of 00f69fd5a9af317bd8000447a4cf50b8 into a33898581a9c4b918a21eab53ce420f8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:58,436 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:58,436 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/B, priority=13, startTime=1732707477989; duration=0sec 2024-11-27T11:37:58,436 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:37:58,436 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:B 2024-11-27T11:37:58,436 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:37:58,437 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:37:58,437 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/C is initiating minor compaction (all files) 2024-11-27T11:37:58,437 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/C in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:58,437 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/53cd01f85a8e4457b945906c6ebf1aa5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b157f4086dad4a06aaad2229befd10fd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/a5d4643f59d14676828769a229fc1109] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=36.2 K 2024-11-27T11:37:58,438 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 53cd01f85a8e4457b945906c6ebf1aa5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1732707474133 2024-11-27T11:37:58,438 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting b157f4086dad4a06aaad2229befd10fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732707475280 2024-11-27T11:37:58,438 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a5d4643f59d14676828769a229fc1109, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732707477468 2024-11-27T11:37:58,443 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:58,444 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/e8f7c4959b964dc8b27268aeba04e133 is 50, key is test_row_0/C:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:58,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742386_1562 (size=12915) 2024-11-27T11:37:58,459 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#A#compaction#467 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:37:58,459 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/31719abee89949e4a1b5d2d45f486393 is 175, key is test_row_0/A:col10/1732707477485/Put/seqid=0 2024-11-27T11:37:58,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742387_1563 (size=31869) 2024-11-27T11:37:58,471 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/a2c2c321db8a4e11a30ae7efdd3f4eb6 2024-11-27T11:37:58,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/bed18951f4344fd78aba79b593b1bca6 is 50, key is test_row_0/B:col10/1732707477522/Put/seqid=0 2024-11-27T11:37:58,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742388_1564 (size=12301) 2024-11-27T11:37:58,482 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/bed18951f4344fd78aba79b593b1bca6 2024-11-27T11:37:58,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/018b13a996c64b3598bcb5250c4cbf3a is 50, key is test_row_0/C:col10/1732707477522/Put/seqid=0 2024-11-27T11:37:58,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742389_1565 (size=12301) 2024-11-27T11:37:58,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707538503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707538503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,506 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707538512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T11:37:58,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707538812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707538812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707538823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:58,855 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/e8f7c4959b964dc8b27268aeba04e133 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/e8f7c4959b964dc8b27268aeba04e133 2024-11-27T11:37:58,859 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/C of 00f69fd5a9af317bd8000447a4cf50b8 into e8f7c4959b964dc8b27268aeba04e133(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:58,859 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:58,859 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/C, priority=13, startTime=1732707477989; duration=0sec 2024-11-27T11:37:58,859 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:58,859 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:C 2024-11-27T11:37:58,869 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/31719abee89949e4a1b5d2d45f486393 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/31719abee89949e4a1b5d2d45f486393 2024-11-27T11:37:58,876 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/A of 00f69fd5a9af317bd8000447a4cf50b8 into 31719abee89949e4a1b5d2d45f486393(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:37:58,876 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:58,876 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/A, priority=13, startTime=1732707477989; duration=0sec 2024-11-27T11:37:58,876 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:37:58,876 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:A 2024-11-27T11:37:58,892 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/018b13a996c64b3598bcb5250c4cbf3a 2024-11-27T11:37:58,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/a2c2c321db8a4e11a30ae7efdd3f4eb6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6 2024-11-27T11:37:58,898 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6, entries=150, sequenceid=284, filesize=30.5 K 2024-11-27T11:37:58,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/bed18951f4344fd78aba79b593b1bca6 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/bed18951f4344fd78aba79b593b1bca6 2024-11-27T11:37:58,902 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/bed18951f4344fd78aba79b593b1bca6, entries=150, sequenceid=284, filesize=12.0 K 2024-11-27T11:37:58,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/018b13a996c64b3598bcb5250c4cbf3a as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/018b13a996c64b3598bcb5250c4cbf3a 2024-11-27T11:37:58,906 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/018b13a996c64b3598bcb5250c4cbf3a, entries=150, sequenceid=284, filesize=12.0 K 2024-11-27T11:37:58,907 INFO [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 00f69fd5a9af317bd8000447a4cf50b8 in 887ms, sequenceid=284, compaction requested=false 2024-11-27T11:37:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:37:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:37:58,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/5b617c7edc8e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-27T11:37:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-27T11:37:58,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-27T11:37:58,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-27T11:37:58,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.5070 sec 2024-11-27T11:37:59,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:37:59,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-27T11:37:59,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279beb21c5b613438c99696265b61c2026_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707478186/Put/seqid=0 2024-11-27T11:37:59,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742390_1566 (size=17534) 2024-11-27T11:37:59,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707539357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707539359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,365 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707539362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,368 INFO [Thread-2198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2198. 2024-11-27T11:37:59,368 DEBUG [Thread-2198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:59,368 DEBUG [Thread-2198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,368 DEBUG [Thread-2198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,368 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:59,369 INFO [Thread-2206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2206. 2024-11-27T11:37:59,370 DEBUG [Thread-2206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:59,370 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,370 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,370 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:59,371 INFO [Thread-2202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2202. 2024-11-27T11:37:59,371 DEBUG [Thread-2202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:59,371 DEBUG [Thread-2202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,371 DEBUG [Thread-2202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:59,371 INFO [Thread-2200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2200. 2024-11-27T11:37:59,371 DEBUG [Thread-2200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:59,372 DEBUG [Thread-2200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,372 DEBUG [Thread-2200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:59,373 INFO [Thread-2204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2204. 2024-11-27T11:37:59,373 DEBUG [Thread-2204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:37:59,373 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,373 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:37:59,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:37:59,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707539473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707539473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707539473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-27T11:37:59,553 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:37:59,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707539682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707539683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707539683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,683 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:37:59,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411279beb21c5b613438c99696265b61c2026_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279beb21c5b613438c99696265b61c2026_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:37:59,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8342d14d57dd491fad68c3d927b6f732, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:37:59,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8342d14d57dd491fad68c3d927b6f732 is 175, key is test_row_0/A:col10/1732707478186/Put/seqid=0 2024-11-27T11:37:59,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742391_1567 (size=48639) 2024-11-27T11:37:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:37:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707539993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707539993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707539993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:37:59,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:38:00,143 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8342d14d57dd491fad68c3d927b6f732 2024-11-27T11:38:00,148 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/320f65afbe224bd4bb6bce64764503a7 is 50, key is test_row_0/B:col10/1732707478186/Put/seqid=0 2024-11-27T11:38:00,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742392_1568 (size=12301) 2024-11-27T11:38:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:38:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:38:00,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-27T11:38:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54608 deadline: 1732707540502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:38:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54636 deadline: 1732707540502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:38:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=35461 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54662 deadline: 1732707540502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 , the old value is region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:38:00,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., hostname=5b617c7edc8e,35461,1732707299005, seqNum=5 because the exception is null or not the one we care about 2024-11-27T11:38:00,551 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/320f65afbe224bd4bb6bce64764503a7 2024-11-27T11:38:00,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/c26afa10a1b5499895206b46f301fa57 is 50, key is test_row_0/C:col10/1732707478186/Put/seqid=0 2024-11-27T11:38:00,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742393_1569 (size=12301) 2024-11-27T11:38:00,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/c26afa10a1b5499895206b46f301fa57 2024-11-27T11:38:00,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/8342d14d57dd491fad68c3d927b6f732 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732 2024-11-27T11:38:00,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732, entries=250, sequenceid=300, filesize=47.5 K 2024-11-27T11:38:00,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/320f65afbe224bd4bb6bce64764503a7 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/320f65afbe224bd4bb6bce64764503a7 2024-11-27T11:38:00,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/320f65afbe224bd4bb6bce64764503a7, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T11:38:00,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/c26afa10a1b5499895206b46f301fa57 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/c26afa10a1b5499895206b46f301fa57 2024-11-27T11:38:00,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/c26afa10a1b5499895206b46f301fa57, entries=150, sequenceid=300, filesize=12.0 K 2024-11-27T11:38:00,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 00f69fd5a9af317bd8000447a4cf50b8 in 1648ms, sequenceid=300, compaction requested=true 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:A, priority=-2147483648, current under compaction store size is 1 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:B, priority=-2147483648, current under compaction store size is 2 2024-11-27T11:38:00,972 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 00f69fd5a9af317bd8000447a4cf50b8:C, priority=-2147483648, current under compaction store size is 3 2024-11-27T11:38:00,972 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:38:00,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111763 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/A is initiating minor compaction (all files) 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/B is initiating minor compaction (all files) 2024-11-27T11:38:00,973 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/A in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:00,973 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/B in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:00,973 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/31719abee89949e4a1b5d2d45f486393, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=109.1 K 2024-11-27T11:38:00,973 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/a33898581a9c4b918a21eab53ce420f8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/bed18951f4344fd78aba79b593b1bca6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/320f65afbe224bd4bb6bce64764503a7] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=36.6 K 2024-11-27T11:38:00,973 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. files: [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/31719abee89949e4a1b5d2d45f486393, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732] 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting a33898581a9c4b918a21eab53ce420f8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732707477468 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31719abee89949e4a1b5d2d45f486393, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732707477468 2024-11-27T11:38:00,973 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting bed18951f4344fd78aba79b593b1bca6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732707477522 2024-11-27T11:38:00,974 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c2c321db8a4e11a30ae7efdd3f4eb6, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732707477522 2024-11-27T11:38:00,974 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 320f65afbe224bd4bb6bce64764503a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732707478186 2024-11-27T11:38:00,974 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8342d14d57dd491fad68c3d927b6f732, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732707478181 2024-11-27T11:38:00,978 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:38:00,979 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:38:00,979 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411278abed0d0a24a46e58245fd17a64fbf9f_00f69fd5a9af317bd8000447a4cf50b8 store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:38:00,979 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/20fe8d2c446e4e4da22e08e66ce996c3 is 50, key is test_row_0/B:col10/1732707478186/Put/seqid=0 2024-11-27T11:38:00,981 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411278abed0d0a24a46e58245fd17a64fbf9f_00f69fd5a9af317bd8000447a4cf50b8, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:38:00,982 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411278abed0d0a24a46e58245fd17a64fbf9f_00f69fd5a9af317bd8000447a4cf50b8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:38:00,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742394_1570 (size=13017) 2024-11-27T11:38:00,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742395_1571 (size=4469) 2024-11-27T11:38:01,386 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#A#compaction#476 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:38:01,387 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/34c6ee3416aa45b99d1770c1ca691fb2 is 175, key is test_row_0/A:col10/1732707478186/Put/seqid=0 2024-11-27T11:38:01,387 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/20fe8d2c446e4e4da22e08e66ce996c3 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/20fe8d2c446e4e4da22e08e66ce996c3 2024-11-27T11:38:01,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742396_1572 (size=31971) 2024-11-27T11:38:01,393 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/B of 00f69fd5a9af317bd8000447a4cf50b8 into 20fe8d2c446e4e4da22e08e66ce996c3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:38:01,393 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:38:01,393 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/B, priority=13, startTime=1732707480972; duration=0sec 2024-11-27T11:38:01,393 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-27T11:38:01,393 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:B 2024-11-27T11:38:01,393 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-27T11:38:01,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-27T11:38:01,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1541): 00f69fd5a9af317bd8000447a4cf50b8/C is initiating minor compaction (all files) 2024-11-27T11:38:01,394 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 00f69fd5a9af317bd8000447a4cf50b8/C in TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:01,394 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/e8f7c4959b964dc8b27268aeba04e133, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/018b13a996c64b3598bcb5250c4cbf3a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/c26afa10a1b5499895206b46f301fa57] into tmpdir=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp, totalSize=36.6 K 2024-11-27T11:38:01,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting e8f7c4959b964dc8b27268aeba04e133, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1732707477468 2024-11-27T11:38:01,394 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting 018b13a996c64b3598bcb5250c4cbf3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732707477522 2024-11-27T11:38:01,395 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] compactions.Compactor(225): Compacting c26afa10a1b5499895206b46f301fa57, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732707478186 2024-11-27T11:38:01,399 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 00f69fd5a9af317bd8000447a4cf50b8#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-27T11:38:01,400 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/2cd4804d1b2c43cb98f6fb19720b1010 is 50, key is test_row_0/C:col10/1732707478186/Put/seqid=0 2024-11-27T11:38:01,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742397_1573 (size=13017) 2024-11-27T11:38:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=35461 {}] regionserver.HRegion(8855): Flush requested on 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:01,515 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-27T11:38:01,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-27T11:38:01,515 INFO [Thread-2193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2193. 2024-11-27T11:38:01,515 INFO [Thread-2191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2191. 2024-11-27T11:38:01,515 INFO [Thread-2195 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2195. 2024-11-27T11:38:01,515 DEBUG [Thread-2191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:38:01,515 DEBUG [Thread-2193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:38:01,515 DEBUG [Thread-2195 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:38:01,515 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:01,515 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:01,515 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:01,515 DEBUG [Thread-2193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:01,515 DEBUG [Thread-2195 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:01,515 DEBUG [Thread-2191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:01,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:01,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:01,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:01,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127824efd0f0d8c484ebded6be5c92469a8_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707481513/Put/seqid=0 2024-11-27T11:38:01,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742398_1574 (size=12454) 2024-11-27T11:38:01,795 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/34c6ee3416aa45b99d1770c1ca691fb2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/34c6ee3416aa45b99d1770c1ca691fb2 2024-11-27T11:38:01,799 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/A of 00f69fd5a9af317bd8000447a4cf50b8 into 34c6ee3416aa45b99d1770c1ca691fb2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:38:01,799 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:38:01,799 INFO [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/A, priority=13, startTime=1732707480972; duration=0sec 2024-11-27T11:38:01,799 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:38:01,799 DEBUG [RS:0;5b617c7edc8e:35461-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:A 2024-11-27T11:38:01,805 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/2cd4804d1b2c43cb98f6fb19720b1010 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/2cd4804d1b2c43cb98f6fb19720b1010 2024-11-27T11:38:01,808 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 00f69fd5a9af317bd8000447a4cf50b8/C of 00f69fd5a9af317bd8000447a4cf50b8 into 2cd4804d1b2c43cb98f6fb19720b1010(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-27T11:38:01,808 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:38:01,809 INFO [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., storeName=00f69fd5a9af317bd8000447a4cf50b8/C, priority=13, startTime=1732707480972; duration=0sec 2024-11-27T11:38:01,809 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-27T11:38:01,809 DEBUG [RS:0;5b617c7edc8e:35461-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 00f69fd5a9af317bd8000447a4cf50b8:C 2024-11-27T11:38:01,924 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:38:01,927 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127824efd0f0d8c484ebded6be5c92469a8_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127824efd0f0d8c484ebded6be5c92469a8_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:01,927 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/de47a97263b4482da1c2ee0997b36ed5, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:38:01,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/de47a97263b4482da1c2ee0997b36ed5 is 175, key is test_row_0/A:col10/1732707481513/Put/seqid=0 2024-11-27T11:38:01,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742399_1575 (size=31255) 2024-11-27T11:38:02,324 INFO [Thread-2189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2189. 2024-11-27T11:38:02,324 DEBUG [Thread-2189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:38:02,324 DEBUG [Thread-2189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:02,324 DEBUG [Thread-2189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:02,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:02,331 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/de47a97263b4482da1c2ee0997b36ed5 2024-11-27T11:38:02,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/f126b41c829e4319a5ebbe942d8f1311 is 50, key is test_row_0/B:col10/1732707481513/Put/seqid=0 2024-11-27T11:38:02,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742400_1576 (size=12301) 2024-11-27T11:38:02,353 INFO [Thread-2187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2187. 2024-11-27T11:38:02,353 DEBUG [Thread-2187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-27T11:38:02,353 DEBUG [Thread-2187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:02,354 DEBUG [Thread-2187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:02,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3544 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3884 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3740 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3548 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3873 2024-11-27T11:38:02,354 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-27T11:38:02,354 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:38:02,354 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:38:02,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:02,354 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:02,354 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:02,355 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-27T11:38:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:02,357 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707482357"}]},"ts":"1732707482357"} 2024-11-27T11:38:02,358 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-27T11:38:02,358 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-27T11:38:02,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-27T11:38:02,360 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, UNASSIGN}] 2024-11-27T11:38:02,361 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, UNASSIGN 2024-11-27T11:38:02,361 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=CLOSING, regionLocation=5b617c7edc8e,35461,1732707299005 2024-11-27T11:38:02,362 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35539 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=CLOSING, location=5b617c7edc8e,35461,1732707299005, table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-27T11:38:02,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=162, ppid=161, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, UNASSIGN because future has completed 2024-11-27T11:38:02,363 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-27T11:38:02,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005}] 2024-11-27T11:38:02,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:02,515 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(122): Close 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:02,515 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-27T11:38:02,515 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1722): Closing 00f69fd5a9af317bd8000447a4cf50b8, disabling compactions & flushes 2024-11-27T11:38:02,515 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:02,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:02,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/f126b41c829e4319a5ebbe942d8f1311 2024-11-27T11:38:02,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/96d46ed3d5864b2aa96ff8f478e27e04 is 50, key is test_row_0/C:col10/1732707481513/Put/seqid=0 2024-11-27T11:38:02,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742401_1577 (size=12301) 2024-11-27T11:38:02,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:03,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/96d46ed3d5864b2aa96ff8f478e27e04 2024-11-27T11:38:03,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/de47a97263b4482da1c2ee0997b36ed5 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/de47a97263b4482da1c2ee0997b36ed5 2024-11-27T11:38:03,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/de47a97263b4482da1c2ee0997b36ed5, entries=150, sequenceid=327, filesize=30.5 K 2024-11-27T11:38:03,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/f126b41c829e4319a5ebbe942d8f1311 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f126b41c829e4319a5ebbe942d8f1311 2024-11-27T11:38:03,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f126b41c829e4319a5ebbe942d8f1311, entries=150, sequenceid=327, filesize=12.0 K 2024-11-27T11:38:03,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/96d46ed3d5864b2aa96ff8f478e27e04 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d46ed3d5864b2aa96ff8f478e27e04 2024-11-27T11:38:03,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d46ed3d5864b2aa96ff8f478e27e04, entries=150, sequenceid=327, filesize=12.0 K 2024-11-27T11:38:03,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=13.42 KB/13740 for 00f69fd5a9af317bd8000447a4cf50b8 in 1647ms, sequenceid=327, compaction requested=false 2024-11-27T11:38:03,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 00f69fd5a9af317bd8000447a4cf50b8: 2024-11-27T11:38:03,162 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:03,162 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:03,162 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. after waiting 0 ms 2024-11-27T11:38:03,162 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:03,162 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(2902): Flushing 00f69fd5a9af317bd8000447a4cf50b8 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-27T11:38:03,167 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eee6123b7288464fafa52545002fae55_00f69fd5a9af317bd8000447a4cf50b8 is 50, key is test_row_0/A:col10/1732707482323/Put/seqid=0 2024-11-27T11:38:03,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742402_1578 (size=9914) 2024-11-27T11:38:03,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:03,570 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-27T11:38:03,573 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241127eee6123b7288464fafa52545002fae55_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eee6123b7288464fafa52545002fae55_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:03,574 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/4787628cedf54549a31ac91c6f748d4c, store: [table=TestAcidGuarantees family=A region=00f69fd5a9af317bd8000447a4cf50b8] 2024-11-27T11:38:03,574 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/4787628cedf54549a31ac91c6f748d4c is 175, key is test_row_0/A:col10/1732707482323/Put/seqid=0 2024-11-27T11:38:03,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742403_1579 (size=22561) 2024-11-27T11:38:03,978 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=334, memsize=4.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/4787628cedf54549a31ac91c6f748d4c 2024-11-27T11:38:03,983 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/209a7eba5d1743e093ed5d026971c63d is 50, key is test_row_0/B:col10/1732707482323/Put/seqid=0 2024-11-27T11:38:03,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742404_1580 (size=9857) 2024-11-27T11:38:04,387 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/209a7eba5d1743e093ed5d026971c63d 2024-11-27T11:38:04,392 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/85a39700f85741df9bdd50f84354e8a2 is 50, key is test_row_0/C:col10/1732707482323/Put/seqid=0 2024-11-27T11:38:04,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742405_1581 (size=9857) 2024-11-27T11:38:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:04,796 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/85a39700f85741df9bdd50f84354e8a2 2024-11-27T11:38:04,799 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/A/4787628cedf54549a31ac91c6f748d4c as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/4787628cedf54549a31ac91c6f748d4c 2024-11-27T11:38:04,802 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/4787628cedf54549a31ac91c6f748d4c, entries=100, sequenceid=334, filesize=22.0 K 2024-11-27T11:38:04,802 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/B/209a7eba5d1743e093ed5d026971c63d as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/209a7eba5d1743e093ed5d026971c63d 2024-11-27T11:38:04,805 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/209a7eba5d1743e093ed5d026971c63d, entries=100, sequenceid=334, filesize=9.6 K 2024-11-27T11:38:04,805 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/.tmp/C/85a39700f85741df9bdd50f84354e8a2 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/85a39700f85741df9bdd50f84354e8a2 2024-11-27T11:38:04,807 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/85a39700f85741df9bdd50f84354e8a2, entries=100, sequenceid=334, filesize=9.6 K 2024-11-27T11:38:04,808 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 00f69fd5a9af317bd8000447a4cf50b8 in 1646ms, sequenceid=334, compaction requested=true 2024-11-27T11:38:04,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/5da951d9a11c42878291f6d8bab38426, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/b96bfdefee0849ed86e6cecaf638fdd7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d08e71a7254c4fbda364f2c56fe0005a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/c2d22e616a674daa86186ba666ff606d, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/31719abee89949e4a1b5d2d45f486393, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732] to archive 2024-11-27T11:38:04,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:38:04,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8cf9976355844aab9ca456b4b58f0ca2 2024-11-27T11:38:04,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/bd114a06ca7e4922a97cb2d25394a6d5 2024-11-27T11:38:04,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ef7b99d98f6041258bff5f853599d807 2024-11-27T11:38:04,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/95f882071bc54dc29924f2f24b34ab82 2024-11-27T11:38:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/59dd21cf09f24a3a9c81919f94f8ee33 2024-11-27T11:38:04,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/6b752013c28442539e8325325233c326 2024-11-27T11:38:04,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/5da951d9a11c42878291f6d8bab38426 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/5da951d9a11c42878291f6d8bab38426 2024-11-27T11:38:04,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/ba5f9a4ccfbd4c919f60ca72ec52b963 2024-11-27T11:38:04,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/b96bfdefee0849ed86e6cecaf638fdd7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/b96bfdefee0849ed86e6cecaf638fdd7 2024-11-27T11:38:04,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/22d9fa6cbd5b4dbd9365d420b4913ca5 2024-11-27T11:38:04,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/2e57e9b9780a4a0781140057b122e6bb 2024-11-27T11:38:04,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e6282f198e8241b4a5796448e597b567 2024-11-27T11:38:04,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d08e71a7254c4fbda364f2c56fe0005a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d08e71a7254c4fbda364f2c56fe0005a 2024-11-27T11:38:04,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e8a2e2d4926149ad950603dd3cea2316 2024-11-27T11:38:04,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/e56c92b94e6a4f7499e643910ab5f34e 2024-11-27T11:38:04,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/c2d22e616a674daa86186ba666ff606d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/c2d22e616a674daa86186ba666ff606d 2024-11-27T11:38:04,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/d8bde88b69774ddf85fb3bf8bdb167db 2024-11-27T11:38:04,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/94cc910a73eb42c6998b899591ad79a8 2024-11-27T11:38:04,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/31719abee89949e4a1b5d2d45f486393 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/31719abee89949e4a1b5d2d45f486393 2024-11-27T11:38:04,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/a2c2c321db8a4e11a30ae7efdd3f4eb6 2024-11-27T11:38:04,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/8342d14d57dd491fad68c3d927b6f732 2024-11-27T11:38:04,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/c80027be68624da0a20732766c7d7e3f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/010d2c5557bd47d8813b632829920d93, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/8ccd33f10e7b470d8f33abf061b37729, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/64b98752554343c1a2ae8c144f53ae94, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/769ba05d32f84e5382516223cd260c7a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b222685bad3d47e6bbda6e4bf16d77fb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/cbd870306cf54350bb885f552255c52a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/0c07b12ed28f4ea39371ef18039ca6d4, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f5bffc26aeff4612905cf26bf5025bb3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b45b430f492546d0af1418f24539f427, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9421e5a1ffdf42569533cf614eaea77a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/7351074c225a4bd9a649bde7fa5ec816, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/add525ba7d934d019cba09ad3bea6cd1, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/3e4d1f4a9cc64e34b30d9ba77be08d50, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/e1fa98240aea4f588c31721030ccf1c6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/2ee357d3347f40d0bf22f02f3f9e343b, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/60b439f20e264195aab8b52c7fa35694, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/a33898581a9c4b918a21eab53ce420f8, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9ec804c2c2554cc79aa9ecc71c8210db, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/bed18951f4344fd78aba79b593b1bca6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/320f65afbe224bd4bb6bce64764503a7] to archive 2024-11-27T11:38:04,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:38:04,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/c80027be68624da0a20732766c7d7e3f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/c80027be68624da0a20732766c7d7e3f 2024-11-27T11:38:04,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/010d2c5557bd47d8813b632829920d93 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/010d2c5557bd47d8813b632829920d93 2024-11-27T11:38:04,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/8ccd33f10e7b470d8f33abf061b37729 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/8ccd33f10e7b470d8f33abf061b37729 2024-11-27T11:38:04,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/64b98752554343c1a2ae8c144f53ae94 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/64b98752554343c1a2ae8c144f53ae94 2024-11-27T11:38:04,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/769ba05d32f84e5382516223cd260c7a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/769ba05d32f84e5382516223cd260c7a 2024-11-27T11:38:04,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b222685bad3d47e6bbda6e4bf16d77fb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b222685bad3d47e6bbda6e4bf16d77fb 2024-11-27T11:38:04,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/cbd870306cf54350bb885f552255c52a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/cbd870306cf54350bb885f552255c52a 2024-11-27T11:38:04,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/0c07b12ed28f4ea39371ef18039ca6d4 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/0c07b12ed28f4ea39371ef18039ca6d4 2024-11-27T11:38:04,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f5bffc26aeff4612905cf26bf5025bb3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f5bffc26aeff4612905cf26bf5025bb3 2024-11-27T11:38:04,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b45b430f492546d0af1418f24539f427 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/b45b430f492546d0af1418f24539f427 2024-11-27T11:38:04,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9421e5a1ffdf42569533cf614eaea77a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9421e5a1ffdf42569533cf614eaea77a 2024-11-27T11:38:04,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/7351074c225a4bd9a649bde7fa5ec816 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/7351074c225a4bd9a649bde7fa5ec816 2024-11-27T11:38:04,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/add525ba7d934d019cba09ad3bea6cd1 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/add525ba7d934d019cba09ad3bea6cd1 2024-11-27T11:38:04,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/3e4d1f4a9cc64e34b30d9ba77be08d50 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/3e4d1f4a9cc64e34b30d9ba77be08d50 2024-11-27T11:38:04,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/e1fa98240aea4f588c31721030ccf1c6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/e1fa98240aea4f588c31721030ccf1c6 2024-11-27T11:38:04,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/2ee357d3347f40d0bf22f02f3f9e343b to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/2ee357d3347f40d0bf22f02f3f9e343b 2024-11-27T11:38:04,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/60b439f20e264195aab8b52c7fa35694 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/60b439f20e264195aab8b52c7fa35694 2024-11-27T11:38:04,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/a33898581a9c4b918a21eab53ce420f8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/a33898581a9c4b918a21eab53ce420f8 2024-11-27T11:38:04,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9ec804c2c2554cc79aa9ecc71c8210db to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/9ec804c2c2554cc79aa9ecc71c8210db 2024-11-27T11:38:04,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/bed18951f4344fd78aba79b593b1bca6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/bed18951f4344fd78aba79b593b1bca6 2024-11-27T11:38:04,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/320f65afbe224bd4bb6bce64764503a7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/320f65afbe224bd4bb6bce64764503a7 2024-11-27T11:38:04,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6bf4c6e929ce4429b43a291388aae5a7, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/1bb07cb83fc4495f98070873557c7072, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6b0541bed29c4e818cbf14cdd4fd993f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/4786a357077d4315baea8f58658b37d6, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/d538bac0183a4b8d8e8caf9bb26db66f, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/79ae9e5435dd46a799d275e0f0ffdef3, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b38eb344096b42a09c7137a4aa39cb21, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/cf8f9400dc464ef1aa675cecdf33b3aa, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/ffd642656cd0455182565a2cf1f0b2ff, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/8f32585098404fb9be9d62b25c60c962, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0506b6e1123b4612a3a78d15954873bb, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0631a58b988c4adebf1ced025b02b9c2, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/dc19a5eec7ae4f22b9c4448abe6aab71, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/bb28377d12e7436d98dbfaf48ca12790, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/53cd01f85a8e4457b945906c6ebf1aa5, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d9ad5b7bfb4bb0bd4dc11dc433650c, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b157f4086dad4a06aaad2229befd10fd, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/e8f7c4959b964dc8b27268aeba04e133, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/a5d4643f59d14676828769a229fc1109, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/018b13a996c64b3598bcb5250c4cbf3a, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/c26afa10a1b5499895206b46f301fa57] to archive 2024-11-27T11:38:04,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-27T11:38:04,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6bf4c6e929ce4429b43a291388aae5a7 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6bf4c6e929ce4429b43a291388aae5a7 2024-11-27T11:38:04,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/1bb07cb83fc4495f98070873557c7072 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/1bb07cb83fc4495f98070873557c7072 2024-11-27T11:38:04,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6b0541bed29c4e818cbf14cdd4fd993f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/6b0541bed29c4e818cbf14cdd4fd993f 2024-11-27T11:38:04,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/4786a357077d4315baea8f58658b37d6 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/4786a357077d4315baea8f58658b37d6 2024-11-27T11:38:04,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/d538bac0183a4b8d8e8caf9bb26db66f to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/d538bac0183a4b8d8e8caf9bb26db66f 2024-11-27T11:38:04,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/79ae9e5435dd46a799d275e0f0ffdef3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/79ae9e5435dd46a799d275e0f0ffdef3 2024-11-27T11:38:04,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b38eb344096b42a09c7137a4aa39cb21 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b38eb344096b42a09c7137a4aa39cb21 2024-11-27T11:38:04,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/cf8f9400dc464ef1aa675cecdf33b3aa to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/cf8f9400dc464ef1aa675cecdf33b3aa 2024-11-27T11:38:04,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/ffd642656cd0455182565a2cf1f0b2ff to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/ffd642656cd0455182565a2cf1f0b2ff 2024-11-27T11:38:04,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/8f32585098404fb9be9d62b25c60c962 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/8f32585098404fb9be9d62b25c60c962 2024-11-27T11:38:04,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0506b6e1123b4612a3a78d15954873bb to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0506b6e1123b4612a3a78d15954873bb 2024-11-27T11:38:04,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0631a58b988c4adebf1ced025b02b9c2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/0631a58b988c4adebf1ced025b02b9c2 2024-11-27T11:38:04,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/dc19a5eec7ae4f22b9c4448abe6aab71 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/dc19a5eec7ae4f22b9c4448abe6aab71 2024-11-27T11:38:04,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/bb28377d12e7436d98dbfaf48ca12790 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/bb28377d12e7436d98dbfaf48ca12790 2024-11-27T11:38:04,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/53cd01f85a8e4457b945906c6ebf1aa5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/53cd01f85a8e4457b945906c6ebf1aa5 2024-11-27T11:38:04,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d9ad5b7bfb4bb0bd4dc11dc433650c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d9ad5b7bfb4bb0bd4dc11dc433650c 2024-11-27T11:38:04,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b157f4086dad4a06aaad2229befd10fd to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/b157f4086dad4a06aaad2229befd10fd 2024-11-27T11:38:04,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/e8f7c4959b964dc8b27268aeba04e133 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/e8f7c4959b964dc8b27268aeba04e133 2024-11-27T11:38:04,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/a5d4643f59d14676828769a229fc1109 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/a5d4643f59d14676828769a229fc1109 2024-11-27T11:38:04,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/018b13a996c64b3598bcb5250c4cbf3a to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/018b13a996c64b3598bcb5250c4cbf3a 2024-11-27T11:38:04,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/c26afa10a1b5499895206b46f301fa57 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/c26afa10a1b5499895206b46f301fa57 2024-11-27T11:38:04,864 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/recovered.edits/337.seqid, newMaxSeqId=337, maxSeqId=4 2024-11-27T11:38:04,864 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. 2024-11-27T11:38:04,864 DEBUG [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] regionserver.HRegion(1676): Region close journal for 00f69fd5a9af317bd8000447a4cf50b8: Waiting for close lock at 1732707482515Running coprocessor pre-close hooks at 1732707482515Disabling compacts and flushes for region at 1732707482515Disabling writes for close at 1732707483162 (+647 ms)Obtaining lock to block concurrent updates at 1732707483162Preparing flush snapshotting stores in 00f69fd5a9af317bd8000447a4cf50b8 at 1732707483162Finished memstore snapshotting TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732707483163 (+1 ms)Flushing stores of TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8. at 1732707483163Flushing 00f69fd5a9af317bd8000447a4cf50b8/A: creating writer at 1732707483163Flushing mob file 00f69fd5a9af317bd8000447a4cf50b8/A: appending metadata at 1732707483167 (+4 ms)Flushing mob file 00f69fd5a9af317bd8000447a4cf50b8/A: closing flushed file at 1732707483167Flushing 00f69fd5a9af317bd8000447a4cf50b8/A: appending metadata at 1732707483574 (+407 ms)Flushing 00f69fd5a9af317bd8000447a4cf50b8/A: closing flushed file at 1732707483574Flushing 00f69fd5a9af317bd8000447a4cf50b8/B: creating writer at 1732707483981 (+407 ms)Flushing 00f69fd5a9af317bd8000447a4cf50b8/B: appending metadata at 1732707483983 (+2 ms)Flushing 00f69fd5a9af317bd8000447a4cf50b8/B: closing flushed file at 1732707483983Flushing 00f69fd5a9af317bd8000447a4cf50b8/C: creating writer at 1732707484390 (+407 ms)Flushing 00f69fd5a9af317bd8000447a4cf50b8/C: appending metadata at 1732707484392 (+2 ms)Flushing 00f69fd5a9af317bd8000447a4cf50b8/C: closing flushed file at 1732707484392Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32d7f42b: reopening flushed file at 1732707484798 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55dbf9fc: reopening flushed file at 1732707484802 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@720491e4: reopening flushed file at 1732707484805 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 00f69fd5a9af317bd8000447a4cf50b8 in 1646ms, sequenceid=334, compaction requested=true at 1732707484808 (+3 ms)Writing region close event to WAL at 1732707484861 (+53 ms)Running coprocessor post-close hooks at 1732707484864 (+3 ms)Closed at 1732707484864 2024-11-27T11:38:04,866 INFO [RS_CLOSE_REGION-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_REGION, pid=163}] handler.UnassignRegionHandler(157): Closed 00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:04,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=162 updating hbase:meta row=00f69fd5a9af317bd8000447a4cf50b8, regionState=CLOSED 2024-11-27T11:38:04,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=163, ppid=162, state=RUNNABLE, hasLock=false; CloseRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 because future has completed 2024-11-27T11:38:04,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-27T11:38:04,869 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseRegionProcedure 00f69fd5a9af317bd8000447a4cf50b8, server=5b617c7edc8e,35461,1732707299005 in 2.5050 sec 2024-11-27T11:38:04,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=162, resume processing ppid=161 2024-11-27T11:38:04,871 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, ppid=161, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=00f69fd5a9af317bd8000447a4cf50b8, UNASSIGN in 2.5090 sec 2024-11-27T11:38:04,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-27T11:38:04,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.5120 sec 2024-11-27T11:38:04,874 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732707484873"}]},"ts":"1732707484873"} 2024-11-27T11:38:04,875 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-27T11:38:04,875 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-27T11:38:04,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5200 sec 2024-11-27T11:38:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-27T11:38:06,512 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:38:06,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-27T11:38:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:06,514 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=164, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:06,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T11:38:06,515 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=164, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:06,517 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,518 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C, FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/recovered.edits] 2024-11-27T11:38:06,520 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/34c6ee3416aa45b99d1770c1ca691fb2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/34c6ee3416aa45b99d1770c1ca691fb2 2024-11-27T11:38:06,521 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/4787628cedf54549a31ac91c6f748d4c to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/4787628cedf54549a31ac91c6f748d4c 2024-11-27T11:38:06,522 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/de47a97263b4482da1c2ee0997b36ed5 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/A/de47a97263b4482da1c2ee0997b36ed5 2024-11-27T11:38:06,523 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/209a7eba5d1743e093ed5d026971c63d to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/209a7eba5d1743e093ed5d026971c63d 2024-11-27T11:38:06,524 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/20fe8d2c446e4e4da22e08e66ce996c3 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/20fe8d2c446e4e4da22e08e66ce996c3 2024-11-27T11:38:06,525 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f126b41c829e4319a5ebbe942d8f1311 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/B/f126b41c829e4319a5ebbe942d8f1311 2024-11-27T11:38:06,526 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/2cd4804d1b2c43cb98f6fb19720b1010 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/2cd4804d1b2c43cb98f6fb19720b1010 2024-11-27T11:38:06,527 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/85a39700f85741df9bdd50f84354e8a2 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/85a39700f85741df9bdd50f84354e8a2 2024-11-27T11:38:06,528 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d46ed3d5864b2aa96ff8f478e27e04 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/C/96d46ed3d5864b2aa96ff8f478e27e04 2024-11-27T11:38:06,530 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/recovered.edits/337.seqid to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8/recovered.edits/337.seqid 2024-11-27T11:38:06,530 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/default/TestAcidGuarantees/00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,530 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-27T11:38:06,531 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T11:38:06,531 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-27T11:38:06,533 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270a947f0639974799b0dccbddbd4a5954_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270a947f0639974799b0dccbddbd4a5954_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,534 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270d37367127364bad80c061ff1466193e_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411270d37367127364bad80c061ff1466193e_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,535 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127213389f5ef754a82bfd9a6cedbb3405b_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127213389f5ef754a82bfd9a6cedbb3405b_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,536 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273a8602959b764d70b54194a801f7c122_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273a8602959b764d70b54194a801f7c122_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,536 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273b74ca87ff2f4d93986fadbb1bcd9dbb_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411273b74ca87ff2f4d93986fadbb1bcd9dbb_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,537 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740da0b5e90364b13bd9490ad8242225a_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112740da0b5e90364b13bd9490ad8242225a_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,538 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112777f450ce2570480980a551fed26a9ef4_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112777f450ce2570480980a551fed26a9ef4_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,539 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127824efd0f0d8c484ebded6be5c92469a8_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127824efd0f0d8c484ebded6be5c92469a8_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,540 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279beb21c5b613438c99696265b61c2026_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279beb21c5b613438c99696265b61c2026_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,540 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279c7fdc79b1a04f82b2bdef298213512d_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279c7fdc79b1a04f82b2bdef298213512d_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,541 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279d5db8092831431fa6b1f870cd47ebd3_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411279d5db8092831431fa6b1f870cd47ebd3_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,542 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a23c877a863748178ede456047485b9e_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a23c877a863748178ede456047485b9e_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,543 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a98e5e9259704308b51fa41890702e32_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127a98e5e9259704308b51fa41890702e32_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,544 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bbc397ead2424d5ba35683fa21ba39dd_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bbc397ead2424d5ba35683fa21ba39dd_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,545 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bd3fcb8d4fb844f1be4183680e449b82_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127bd3fcb8d4fb844f1be4183680e449b82_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,546 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dda0d40780a94e908cc76aa78b11c9f2_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127dda0d40780a94e908cc76aa78b11c9f2_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,547 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127df69b41ccdf44dedbb09073f82047e02_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127df69b41ccdf44dedbb09073f82047e02_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,547 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eee6123b7288464fafa52545002fae55_00f69fd5a9af317bd8000447a4cf50b8 to hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241127eee6123b7288464fafa52545002fae55_00f69fd5a9af317bd8000447a4cf50b8 2024-11-27T11:38:06,548 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-27T11:38:06,550 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=164, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:06,551 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-27T11:38:06,553 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-27T11:38:06,554 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=164, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:06,554 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-27T11:38:06,554 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732707486554"}]},"ts":"9223372036854775807"} 2024-11-27T11:38:06,555 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-27T11:38:06,555 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 00f69fd5a9af317bd8000447a4cf50b8, NAME => 'TestAcidGuarantees,,1732707456260.00f69fd5a9af317bd8000447a4cf50b8.', STARTKEY => '', ENDKEY => ''}] 2024-11-27T11:38:06,556 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-27T11:38:06,556 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732707486556"}]},"ts":"9223372036854775807"} 2024-11-27T11:38:06,557 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-27T11:38:06,557 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=164, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-27T11:38:06,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 44 msec 2024-11-27T11:38:06,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35539 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-27T11:38:06,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-27T11:38:06,622 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-27T11:38:06,632 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 236) - Thread LEAK? -, OpenFileDescriptor=455 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=247 (was 227) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6328 (was 6336) 2024-11-27T11:38:06,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-27T11:38:06,632 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:38:06,632 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:38:06,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:06,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:06,632 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-27T11:38:06,633 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-27T11:38:06,633 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1603076298, stopped=false 2024-11-27T11:38:06,633 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=5b617c7edc8e,35539,1732707298302 2024-11-27T11:38:06,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T11:38:06,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-27T11:38:06,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:38:06,635 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:38:06,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T11:38:06,635 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-27T11:38:06,635 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:38:06,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:06,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T11:38:06,636 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-27T11:38:06,636 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '5b617c7edc8e,35461,1732707299005' ***** 2024-11-27T11:38:06,636 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-27T11:38:06,636 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-27T11:38:06,637 INFO [RS:0;5b617c7edc8e:35461 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-27T11:38:06,637 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-27T11:38:06,637 INFO [RS:0;5b617c7edc8e:35461 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-27T11:38:06,637 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(959): stopping server 5b617c7edc8e,35461,1732707299005 2024-11-27T11:38:06,637 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T11:38:06,637 INFO [RS:0;5b617c7edc8e:35461 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;5b617c7edc8e:35461. 2024-11-27T11:38:06,637 DEBUG [RS:0;5b617c7edc8e:35461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-27T11:38:06,637 DEBUG [RS:0;5b617c7edc8e:35461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:06,638 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-27T11:38:06,638 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-27T11:38:06,638 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-27T11:38:06,638 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-27T11:38:06,638 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-27T11:38:06,638 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-27T11:38:06,639 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-27T11:38:06,639 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-27T11:38:06,639 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-27T11:38:06,639 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-27T11:38:06,639 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-27T11:38:06,639 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-27T11:38:06,639 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:06,662 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/info/cdf7fb97076448b486060a2a55354038 is 95, key is TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b./info:/1732707328723/DeleteFamily/seqid=0 2024-11-27T11:38:06,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742406_1582 (size=6514) 2024-11-27T11:38:06,667 INFO [regionserver/5b617c7edc8e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T11:38:06,801 INFO [regionserver/5b617c7edc8e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-27T11:38:06,802 INFO [regionserver/5b617c7edc8e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-27T11:38:06,839 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:07,039 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:07,066 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/info/cdf7fb97076448b486060a2a55354038 2024-11-27T11:38:07,083 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/ns/af8b49141fb744d6baf546f96614b8bc is 93, key is TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b./ns:/1732707328682/DeleteFamily/seqid=0 2024-11-27T11:38:07,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742407_1583 (size=5906) 2024-11-27T11:38:07,240 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:07,440 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:07,487 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/ns/af8b49141fb744d6baf546f96614b8bc 2024-11-27T11:38:07,504 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/rep_barrier/10f3ae0bd5c04d908774b9ccb5ec0556 is 102, key is TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b./rep_barrier:/1732707328682/DeleteFamily/seqid=0 2024-11-27T11:38:07,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742408_1584 (size=6025) 2024-11-27T11:38:07,640 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:07,840 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:07,907 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/rep_barrier/10f3ae0bd5c04d908774b9ccb5ec0556 2024-11-27T11:38:07,925 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/table/ac84435f6bf64270aa81ff94a83bad9e is 96, key is TestAcidGuarantees,,1732707303324.080ce2238710db49165afb3a1a396a6b./table:/1732707328682/DeleteFamily/seqid=0 2024-11-27T11:38:07,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742409_1585 (size=5936) 2024-11-27T11:38:08,040 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:08,240 DEBUG [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-27T11:38:08,328 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/table/ac84435f6bf64270aa81ff94a83bad9e 2024-11-27T11:38:08,332 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/info/cdf7fb97076448b486060a2a55354038 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/info/cdf7fb97076448b486060a2a55354038 2024-11-27T11:38:08,334 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/info/cdf7fb97076448b486060a2a55354038, entries=12, sequenceid=90, filesize=6.4 K 2024-11-27T11:38:08,335 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/ns/af8b49141fb744d6baf546f96614b8bc as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/ns/af8b49141fb744d6baf546f96614b8bc 2024-11-27T11:38:08,337 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/ns/af8b49141fb744d6baf546f96614b8bc, entries=8, sequenceid=90, filesize=5.8 K 2024-11-27T11:38:08,338 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/rep_barrier/10f3ae0bd5c04d908774b9ccb5ec0556 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/rep_barrier/10f3ae0bd5c04d908774b9ccb5ec0556 2024-11-27T11:38:08,341 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/rep_barrier/10f3ae0bd5c04d908774b9ccb5ec0556, entries=6, sequenceid=90, filesize=5.9 K 2024-11-27T11:38:08,341 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/.tmp/table/ac84435f6bf64270aa81ff94a83bad9e as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/table/ac84435f6bf64270aa81ff94a83bad9e 2024-11-27T11:38:08,344 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/table/ac84435f6bf64270aa81ff94a83bad9e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-27T11:38:08,345 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false 2024-11-27T11:38:08,349 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-27T11:38:08,349 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T11:38:08,349 INFO [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-27T11:38:08,349 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732707486638Running coprocessor pre-close hooks at 1732707486639 (+1 ms)Disabling compacts and flushes for region at 1732707486639Disabling writes for close at 1732707486639Obtaining lock to block concurrent updates at 1732707486639Preparing flush snapshotting stores in 1588230740 at 1732707486639Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732707486639Flushing stores of hbase:meta,,1.1588230740 at 1732707486640 (+1 ms)Flushing 1588230740/info: creating writer at 1732707486640Flushing 1588230740/info: appending metadata at 1732707486662 (+22 ms)Flushing 1588230740/info: closing flushed file at 1732707486662Flushing 1588230740/ns: creating writer at 1732707487069 (+407 ms)Flushing 1588230740/ns: appending metadata at 1732707487083 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732707487083Flushing 1588230740/rep_barrier: creating writer at 1732707487490 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732707487504 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732707487504Flushing 1588230740/table: creating writer at 1732707487910 (+406 ms)Flushing 1588230740/table: appending metadata at 1732707487924 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732707487924Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2426a2db: reopening flushed file at 1732707488331 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@333de877: reopening flushed file at 1732707488334 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6344eed2: reopening flushed file at 1732707488337 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1149e40f: reopening flushed file at 1732707488341 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1706ms, sequenceid=90, compaction requested=false at 1732707488345 (+4 ms)Writing region close event to WAL at 1732707488346 (+1 ms)Running coprocessor post-close hooks at 1732707488349 (+3 ms)Closed at 1732707488349 2024-11-27T11:38:08,349 DEBUG [RS_CLOSE_META-regionserver/5b617c7edc8e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-27T11:38:08,441 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(976): stopping server 5b617c7edc8e,35461,1732707299005; all regions closed. 2024-11-27T11:38:08,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741834_1010 (size=25344) 2024-11-27T11:38:08,447 DEBUG [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/oldWALs 2024-11-27T11:38:08,447 INFO [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5b617c7edc8e%2C35461%2C1732707299005.meta:.meta(num 1732707302857) 2024-11-27T11:38:08,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741832_1008 (size=12961124) 2024-11-27T11:38:08,451 DEBUG [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/oldWALs 2024-11-27T11:38:08,451 INFO [RS:0;5b617c7edc8e:35461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 5b617c7edc8e%2C35461%2C1732707299005:(num 1732707301961) 2024-11-27T11:38:08,451 DEBUG [RS:0;5b617c7edc8e:35461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-27T11:38:08,451 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.LeaseManager(133): Closed leases 2024-11-27T11:38:08,451 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T11:38:08,451 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.ChoreService(370): Chore service for: regionserver/5b617c7edc8e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-27T11:38:08,451 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T11:38:08,451 INFO [regionserver/5b617c7edc8e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T11:38:08,452 INFO [RS:0;5b617c7edc8e:35461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35461 2024-11-27T11:38:08,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/5b617c7edc8e,35461,1732707299005 2024-11-27T11:38:08,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-27T11:38:08,455 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T11:38:08,457 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [5b617c7edc8e,35461,1732707299005] 2024-11-27T11:38:08,460 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/5b617c7edc8e,35461,1732707299005 already deleted, retry=false 2024-11-27T11:38:08,461 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 5b617c7edc8e,35461,1732707299005 expired; onlineServers=0 2024-11-27T11:38:08,461 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '5b617c7edc8e,35539,1732707298302' ***** 2024-11-27T11:38:08,461 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-27T11:38:08,461 INFO [M:0;5b617c7edc8e:35539 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-27T11:38:08,461 INFO [M:0;5b617c7edc8e:35539 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-27T11:38:08,461 DEBUG [M:0;5b617c7edc8e:35539 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-27T11:38:08,461 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-27T11:38:08,461 DEBUG [M:0;5b617c7edc8e:35539 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-27T11:38:08,461 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster-HFileCleaner.small.0-1732707301604 {}] cleaner.HFileCleaner(306): Exit Thread[master/5b617c7edc8e:0:becomeActiveMaster-HFileCleaner.small.0-1732707301604,5,FailOnTimeoutGroup] 2024-11-27T11:38:08,461 DEBUG [master/5b617c7edc8e:0:becomeActiveMaster-HFileCleaner.large.0-1732707301602 {}] cleaner.HFileCleaner(306): Exit Thread[master/5b617c7edc8e:0:becomeActiveMaster-HFileCleaner.large.0-1732707301602,5,FailOnTimeoutGroup] 2024-11-27T11:38:08,461 INFO [M:0;5b617c7edc8e:35539 {}] hbase.ChoreService(370): Chore service for: master/5b617c7edc8e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-27T11:38:08,462 INFO [M:0;5b617c7edc8e:35539 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-27T11:38:08,462 DEBUG [M:0;5b617c7edc8e:35539 {}] master.HMaster(1795): Stopping service threads 2024-11-27T11:38:08,462 INFO [M:0;5b617c7edc8e:35539 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-27T11:38:08,462 INFO [M:0;5b617c7edc8e:35539 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-27T11:38:08,462 ERROR [M:0;5b617c7edc8e:35539 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-27T11:38:08,462 INFO [M:0;5b617c7edc8e:35539 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-27T11:38:08,463 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-27T11:38:08,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-27T11:38:08,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-27T11:38:08,463 DEBUG [M:0;5b617c7edc8e:35539 {}] zookeeper.ZKUtil(347): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-27T11:38:08,463 WARN [M:0;5b617c7edc8e:35539 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-27T11:38:08,464 INFO [M:0;5b617c7edc8e:35539 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/.lastflushedseqids 2024-11-27T11:38:08,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742410_1586 (size=72) 2024-11-27T11:38:08,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T11:38:08,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35461-0x101593513670001, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T11:38:08,558 INFO [RS:0;5b617c7edc8e:35461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T11:38:08,558 INFO [RS:0;5b617c7edc8e:35461 {}] regionserver.HRegionServer(1031): Exiting; stopping=5b617c7edc8e,35461,1732707299005; zookeeper connection closed. 2024-11-27T11:38:08,559 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@2ec5a0af {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@2ec5a0af 2024-11-27T11:38:08,559 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-27T11:38:08,705 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-27T11:38:08,705 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-27T11:38:08,706 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-27T11:38:08,873 INFO [M:0;5b617c7edc8e:35539 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-27T11:38:08,873 INFO [M:0;5b617c7edc8e:35539 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-27T11:38:08,873 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-27T11:38:08,874 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:38:08,874 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:38:08,874 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-27T11:38:08,874 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:38:08,874 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=719.52 KB heapSize=882.16 KB 2024-11-27T11:38:08,889 DEBUG [M:0;5b617c7edc8e:35539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c1992ea727643929dc35e0c37ec4432 is 82, key is hbase:meta,,1/info:regioninfo/1732707302942/Put/seqid=0 2024-11-27T11:38:08,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742411_1587 (size=5672) 2024-11-27T11:38:09,293 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1998 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c1992ea727643929dc35e0c37ec4432 2024-11-27T11:38:09,312 DEBUG [M:0;5b617c7edc8e:35539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7cf78e79ba9044de8896f268968d49d0 is 2275, key is \x00\x00\x00\x00\x00\x00\x00\x87/proc:d/1732707459335/Put/seqid=0 2024-11-27T11:38:09,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742412_1588 (size=43073) 2024-11-27T11:38:09,716 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=718.97 KB at sequenceid=1998 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7cf78e79ba9044de8896f268968d49d0 2024-11-27T11:38:09,719 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7cf78e79ba9044de8896f268968d49d0 2024-11-27T11:38:09,733 DEBUG [M:0;5b617c7edc8e:35539 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ddb9164cb6954e6493dfb60852c9c435 is 69, key is 5b617c7edc8e,35461,1732707299005/rs:state/1732707301733/Put/seqid=0 2024-11-27T11:38:09,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073742413_1589 (size=5156) 2024-11-27T11:38:10,137 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1998 (bloomFilter=true), to=hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ddb9164cb6954e6493dfb60852c9c435 2024-11-27T11:38:10,140 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2c1992ea727643929dc35e0c37ec4432 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c1992ea727643929dc35e0c37ec4432 2024-11-27T11:38:10,143 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2c1992ea727643929dc35e0c37ec4432, entries=8, sequenceid=1998, filesize=5.5 K 2024-11-27T11:38:10,143 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7cf78e79ba9044de8896f268968d49d0 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7cf78e79ba9044de8896f268968d49d0 2024-11-27T11:38:10,146 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7cf78e79ba9044de8896f268968d49d0 2024-11-27T11:38:10,146 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7cf78e79ba9044de8896f268968d49d0, entries=164, sequenceid=1998, filesize=42.1 K 2024-11-27T11:38:10,146 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/ddb9164cb6954e6493dfb60852c9c435 as hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ddb9164cb6954e6493dfb60852c9c435 2024-11-27T11:38:10,149 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42541/user/jenkins/test-data/691c294f-9dc3-7cef-8e41-7afd9a4f005b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/ddb9164cb6954e6493dfb60852c9c435, entries=1, sequenceid=1998, filesize=5.0 K 2024-11-27T11:38:10,149 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(3140): Finished flush of dataSize ~719.52 KB/736791, heapSize ~881.87 KB/903032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=1998, compaction requested=false 2024-11-27T11:38:10,151 INFO [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-27T11:38:10,151 DEBUG [M:0;5b617c7edc8e:35539 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732707488873Disabling compacts and flushes for region at 1732707488873Disabling writes for close at 1732707488874 (+1 ms)Obtaining lock to block concurrent updates at 1732707488874Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732707488874Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=736791, getHeapSize=903272, getOffHeapSize=0, getCellsCount=2187 at 1732707488874Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732707488874Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732707488874Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732707488889 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732707488889Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732707489296 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732707489311 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732707489311Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732707489719 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732707489733 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732707489733Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f0e6921: reopening flushed file at 1732707490139 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25fe9eb8: reopening flushed file at 1732707490143 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11ce2c84: reopening flushed file at 1732707490146 (+3 ms)Finished flush of dataSize ~719.52 KB/736791, heapSize ~881.87 KB/903032, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=1998, compaction requested=false at 1732707490149 (+3 ms)Writing region close event to WAL at 1732707490151 (+2 ms)Closed at 1732707490151 2024-11-27T11:38:10,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37081 is added to blk_1073741830_1006 (size=867836) 2024-11-27T11:38:10,154 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-27T11:38:10,154 INFO [M:0;5b617c7edc8e:35539 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-27T11:38:10,154 INFO [M:0;5b617c7edc8e:35539 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35539 2024-11-27T11:38:10,154 INFO [M:0;5b617c7edc8e:35539 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-27T11:38:10,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T11:38:10,256 INFO [M:0;5b617c7edc8e:35539 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-27T11:38:10,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35539-0x101593513670000, quorum=127.0.0.1:50523, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-27T11:38:10,261 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@486706b6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-27T11:38:10,263 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1889dd8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T11:38:10,264 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T11:38:10,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33cacd2f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T11:38:10,264 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5afe3a9{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/hadoop.log.dir/,STOPPED} 2024-11-27T11:38:10,266 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-27T11:38:10,266 WARN [BP-972261524-172.17.0.2-1732707295566 heartbeating to localhost/127.0.0.1:42541 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-27T11:38:10,266 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-27T11:38:10,266 WARN [BP-972261524-172.17.0.2-1732707295566 heartbeating to localhost/127.0.0.1:42541 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-972261524-172.17.0.2-1732707295566 (Datanode Uuid b9f6a65c-eef6-4672-a6a3-56fdb08a19fe) service to localhost/127.0.0.1:42541 2024-11-27T11:38:10,268 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/data/data1/current/BP-972261524-172.17.0.2-1732707295566 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T11:38:10,268 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/cluster_b4fb454b-35cf-a15a-146f-cc6f8e91654d/data/data2/current/BP-972261524-172.17.0.2-1732707295566 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-27T11:38:10,269 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-27T11:38:10,275 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3654837{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-27T11:38:10,276 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3edc1d8c{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-27T11:38:10,276 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-27T11:38:10,276 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4a4bbe01{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-27T11:38:10,276 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@493a7020{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/84c04682-7045-b4df-cca1-47a912aa9059/hadoop.log.dir/,STOPPED} 2024-11-27T11:38:10,292 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-27T11:38:10,376 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-27T11:38:10,378 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-27T11:38:10,414 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down